2024-11-28 07:40:29,773 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 07:40:29,790 main DEBUG Took 0.012649 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 07:40:29,791 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 07:40:29,791 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 07:40:29,792 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 07:40:29,794 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,803 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 07:40:29,819 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,821 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,823 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,824 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,825 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,827 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,828 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,829 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,830 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,831 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,832 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,833 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 07:40:29,834 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,834 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 07:40:29,836 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 07:40:29,838 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 07:40:29,845 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 07:40:29,851 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 07:40:29,852 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 07:40:29,853 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 07:40:29,867 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 07:40:29,871 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 07:40:29,874 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 07:40:29,874 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 07:40:29,876 main DEBUG createAppenders(={Console}) 2024-11-28 07:40:29,878 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-28 07:40:29,878 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-28 07:40:29,878 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-28 07:40:29,879 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 07:40:29,879 main DEBUG OutputStream closed 2024-11-28 07:40:29,880 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 07:40:29,880 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 07:40:29,881 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-28 07:40:29,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 07:40:29,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 07:40:29,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 07:40:29,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 07:40:29,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 07:40:29,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 07:40:29,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 07:40:29,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 07:40:29,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 07:40:29,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 07:40:29,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 07:40:29,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 07:40:29,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 07:40:29,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 07:40:29,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 07:40:29,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 07:40:29,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 07:40:29,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 07:40:29,986 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 07:40:29,986 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-28 07:40:29,987 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 07:40:29,989 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-28T07:40:30,337 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c 2024-11-28 07:40:30,341 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 07:40:30,341 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T07:40:30,353 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-28T07:40:30,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T07:40:30,398 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4, deleteOnExit=true 2024-11-28T07:40:30,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T07:40:30,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/test.cache.data in system properties and HBase conf 2024-11-28T07:40:30,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T07:40:30,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/hadoop.log.dir in system properties and HBase conf 2024-11-28T07:40:30,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T07:40:30,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T07:40:30,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T07:40:30,525 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T07:40:30,661 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T07:40:30,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T07:40:30,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T07:40:30,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T07:40:30,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T07:40:30,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T07:40:30,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T07:40:30,681 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T07:40:30,682 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T07:40:30,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T07:40:30,683 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/nfs.dump.dir in system properties and HBase conf 2024-11-28T07:40:30,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/java.io.tmpdir in system properties and HBase conf 2024-11-28T07:40:30,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T07:40:30,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T07:40:30,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T07:40:31,652 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T07:40:31,738 INFO [Time-limited test {}] log.Log(170): Logging initialized @2909ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T07:40:31,817 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:40:31,887 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T07:40:31,912 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T07:40:31,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T07:40:31,914 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T07:40:31,928 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:40:31,930 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/hadoop.log.dir/,AVAILABLE} 2024-11-28T07:40:31,931 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T07:40:32,155 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/java.io.tmpdir/jetty-localhost-36363-hadoop-hdfs-3_4_1-tests_jar-_-any-9036461692231014816/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T07:40:32,162 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:36363} 2024-11-28T07:40:32,163 INFO [Time-limited test {}] server.Server(415): Started @3334ms 2024-11-28T07:40:32,606 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T07:40:32,614 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T07:40:32,615 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T07:40:32,616 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T07:40:32,616 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T07:40:32,617 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/hadoop.log.dir/,AVAILABLE} 2024-11-28T07:40:32,618 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T07:40:32,741 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/java.io.tmpdir/jetty-localhost-37155-hadoop-hdfs-3_4_1-tests_jar-_-any-14380206688678162585/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T07:40:32,742 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:37155} 2024-11-28T07:40:32,742 INFO [Time-limited test {}] server.Server(415): Started @3913ms 2024-11-28T07:40:32,799 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T07:40:33,383 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/data/data2/current/BP-123607211-172.17.0.2-1732779631377/current, will proceed with Du for space computation calculation, 2024-11-28T07:40:33,385 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/data/data1/current/BP-123607211-172.17.0.2-1732779631377/current, will proceed with Du for space computation calculation, 2024-11-28T07:40:33,458 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T07:40:33,527 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd53200d7b141783c with lease ID 0x8da8edfc779501e1: Processing first storage report for DS-c3cb718e-e593-4a50-82ce-5ce19113defd from datanode DatanodeRegistration(127.0.0.1:36239, datanodeUuid=03c664ec-2777-4875-979d-320ad1fc2a6f, infoPort=37517, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1245120071;c=1732779631377) 2024-11-28T07:40:33,529 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd53200d7b141783c with lease ID 0x8da8edfc779501e1: from storage DS-c3cb718e-e593-4a50-82ce-5ce19113defd node DatanodeRegistration(127.0.0.1:36239, datanodeUuid=03c664ec-2777-4875-979d-320ad1fc2a6f, infoPort=37517, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1245120071;c=1732779631377), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-28T07:40:33,530 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd53200d7b141783c with lease ID 0x8da8edfc779501e1: Processing first storage report for DS-6ac34359-b1fc-4616-ae8f-a3ba454af04c from datanode DatanodeRegistration(127.0.0.1:36239, datanodeUuid=03c664ec-2777-4875-979d-320ad1fc2a6f, infoPort=37517, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1245120071;c=1732779631377) 2024-11-28T07:40:33,530 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd53200d7b141783c with lease ID 0x8da8edfc779501e1: from storage DS-6ac34359-b1fc-4616-ae8f-a3ba454af04c node DatanodeRegistration(127.0.0.1:36239, datanodeUuid=03c664ec-2777-4875-979d-320ad1fc2a6f, infoPort=37517, infoSecurePort=0, ipcPort=37855, storageInfo=lv=-57;cid=testClusterID;nsid=1245120071;c=1732779631377), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T07:40:33,592 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c 2024-11-28T07:40:33,711 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/zookeeper_0, clientPort=53972, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T07:40:33,726 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53972 2024-11-28T07:40:33,752 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:40:33,756 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:40:34,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741825_1001 (size=7) 2024-11-28T07:40:34,500 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 with version=8 2024-11-28T07:40:34,501 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/hbase-staging 2024-11-28T07:40:34,621 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T07:40:34,879 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:40:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:40:34,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:40:34,895 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:40:34,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:40:34,896 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:40:35,049 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T07:40:35,117 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T07:40:35,129 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T07:40:35,134 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:40:35,162 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1557 (auto-detected) 2024-11-28T07:40:35,164 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T07:40:35,182 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39063 2024-11-28T07:40:35,205 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39063 connecting to ZooKeeper ensemble=127.0.0.1:53972 2024-11-28T07:40:35,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390630x0, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:40:35,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39063-0x1015d84d6970000 connected 2024-11-28T07:40:35,470 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:40:35,472 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:40:35,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:40:35,486 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573, hbase.cluster.distributed=false 2024-11-28T07:40:35,515 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:40:35,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39063 2024-11-28T07:40:35,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39063 2024-11-28T07:40:35,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39063 2024-11-28T07:40:35,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39063 2024-11-28T07:40:35,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39063 2024-11-28T07:40:35,667 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f64cf8bdf19b:0 server-side Connection retries=45 2024-11-28T07:40:35,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:40:35,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T07:40:35,670 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T07:40:35,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T07:40:35,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T07:40:35,674 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T07:40:35,677 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T07:40:35,678 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34437 2024-11-28T07:40:35,681 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34437 connecting to ZooKeeper ensemble=127.0.0.1:53972 2024-11-28T07:40:35,683 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:40:35,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:40:35,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344370x0, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T07:40:35,695 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:344370x0, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:40:35,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34437-0x1015d84d6970001 connected 2024-11-28T07:40:35,701 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T07:40:35,711 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T07:40:35,715 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T07:40:35,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T07:40:35,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34437 2024-11-28T07:40:35,728 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34437 2024-11-28T07:40:35,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34437 2024-11-28T07:40:35,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34437 2024-11-28T07:40:35,734 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34437 2024-11-28T07:40:35,756 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f64cf8bdf19b:39063 2024-11-28T07:40:35,759 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:35,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:40:35,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:40:35,770 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:35,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:35,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T07:40:35,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:35,798 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T07:40:35,800 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f64cf8bdf19b,39063,1732779634677 from backup master directory 2024-11-28T07:40:35,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:35,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:40:35,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T07:40:35,804 WARN [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:40:35,805 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:35,807 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T07:40:35,808 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T07:40:35,873 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/hbase.id] with ID: 7ca35be7-5931-4a41-bd2b-792d46dc0434 2024-11-28T07:40:35,873 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/.tmp/hbase.id 2024-11-28T07:40:35,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741826_1002 (size=42) 2024-11-28T07:40:36,286 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/.tmp/hbase.id]:[hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/hbase.id] 2024-11-28T07:40:36,334 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T07:40:36,339 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T07:40:36,362 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-28T07:40:36,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:36,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:36,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741827_1003 (size=196) 2024-11-28T07:40:36,401 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:40:36,407 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T07:40:36,429 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:36,436 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:40:36,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741828_1004 (size=1189) 2024-11-28T07:40:36,895 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store 2024-11-28T07:40:36,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741829_1005 (size=34) 2024-11-28T07:40:36,923 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T07:40:36,927 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:40:36,929 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:40:36,929 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:40:36,929 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:40:36,931 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:40:36,932 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:40:36,932 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:40:36,934 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732779636928Disabling compacts and flushes for region at 1732779636928Disabling writes for close at 1732779636931 (+3 ms)Writing region close event to WAL at 1732779636932 (+1 ms)Closed at 1732779636932 2024-11-28T07:40:36,937 WARN [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/.initializing 2024-11-28T07:40:36,937 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/WALs/f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:36,948 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:40:36,965 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f64cf8bdf19b%2C39063%2C1732779634677, suffix=, logDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/WALs/f64cf8bdf19b,39063,1732779634677, archiveDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/oldWALs, maxLogs=10 2024-11-28T07:40:36,995 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/WALs/f64cf8bdf19b,39063,1732779634677/f64cf8bdf19b%2C39063%2C1732779634677.1732779636971, exclude list is [], retry=0 2024-11-28T07:40:37,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36239,DS-c3cb718e-e593-4a50-82ce-5ce19113defd,DISK] 2024-11-28T07:40:37,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T07:40:37,058 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/WALs/f64cf8bdf19b,39063,1732779634677/f64cf8bdf19b%2C39063%2C1732779634677.1732779636971 2024-11-28T07:40:37,059 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37517:37517)] 2024-11-28T07:40:37,060 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:40:37,061 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:40:37,064 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,065 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T07:40:37,145 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:37,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:37,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T07:40:37,153 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:37,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:40:37,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T07:40:37,158 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:37,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:40:37,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T07:40:37,163 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:37,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:40:37,165 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,169 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,171 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,177 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,177 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,182 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T07:40:37,186 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T07:40:37,192 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:40:37,193 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68923082, jitterRate=0.027033954858779907}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T07:40:37,200 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732779637083Initializing all the Stores at 1732779637085 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779637086 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779637087 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779637087Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779637087Cleaning up temporary data from old regions at 1732779637177 (+90 ms)Region opened successfully at 1732779637200 (+23 ms) 2024-11-28T07:40:37,202 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T07:40:37,242 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711e7eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:40:37,277 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T07:40:37,290 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T07:40:37,290 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T07:40:37,294 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T07:40:37,295 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T07:40:37,301 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-28T07:40:37,301 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T07:40:37,335 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T07:40:37,346 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T07:40:37,348 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T07:40:37,352 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T07:40:37,354 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T07:40:37,355 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T07:40:37,358 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T07:40:37,363 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T07:40:37,365 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T07:40:37,367 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T07:40:37,369 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T07:40:37,391 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T07:40:37,393 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T07:40:37,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:40:37,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T07:40:37,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:37,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:37,399 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f64cf8bdf19b,39063,1732779634677, sessionid=0x1015d84d6970000, setting cluster-up flag (Was=false) 2024-11-28T07:40:37,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:37,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:37,418 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T07:40:37,420 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:37,433 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T07:40:37,435 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:37,442 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T07:40:37,520 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T07:40:37,530 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T07:40:37,536 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T07:40:37,539 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(746): ClusterId : 7ca35be7-5931-4a41-bd2b-792d46dc0434 2024-11-28T07:40:37,542 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T07:40:37,543 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f64cf8bdf19b,39063,1732779634677 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T07:40:37,547 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T07:40:37,547 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T07:40:37,550 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T07:40:37,551 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:40:37,551 DEBUG [RS:0;f64cf8bdf19b:34437 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e1c937, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f64cf8bdf19b/172.17.0.2:0 2024-11-28T07:40:37,551 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:40:37,551 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:40:37,551 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=5, maxPoolSize=5 2024-11-28T07:40:37,551 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f64cf8bdf19b:0, corePoolSize=10, maxPoolSize=10 2024-11-28T07:40:37,552 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,552 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:40:37,552 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,555 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732779667555 2024-11-28T07:40:37,557 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T07:40:37,558 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:40:37,558 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T07:40:37,558 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T07:40:37,562 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T07:40:37,562 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T07:40:37,563 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T07:40:37,563 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T07:40:37,563 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,565 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:37,565 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T07:40:37,567 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T07:40:37,568 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T07:40:37,568 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T07:40:37,569 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f64cf8bdf19b:34437 2024-11-28T07:40:37,570 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T07:40:37,571 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T07:40:37,573 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779637572,5,FailOnTimeoutGroup] 2024-11-28T07:40:37,573 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T07:40:37,573 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T07:40:37,573 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T07:40:37,574 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779637573,5,FailOnTimeoutGroup] 2024-11-28T07:40:37,574 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,574 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T07:40:37,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741831_1007 (size=1321) 2024-11-28T07:40:37,575 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,576 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,576 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(2659): reportForDuty to master=f64cf8bdf19b,39063,1732779634677 with port=34437, startcode=1732779635619 2024-11-28T07:40:37,587 DEBUG [RS:0;f64cf8bdf19b:34437 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T07:40:37,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42939, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T07:40:37,631 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39063 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:37,633 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39063 {}] master.ServerManager(517): Registering regionserver=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:37,649 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:40:37,650 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44369 2024-11-28T07:40:37,650 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T07:40:37,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:40:37,655 DEBUG [RS:0;f64cf8bdf19b:34437 {}] zookeeper.ZKUtil(111): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:37,655 WARN [RS:0;f64cf8bdf19b:34437 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T07:40:37,655 INFO [RS:0;f64cf8bdf19b:34437 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:40:37,655 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/WALs/f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:37,658 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f64cf8bdf19b,34437,1732779635619] 2024-11-28T07:40:37,683 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T07:40:37,695 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T07:40:37,700 INFO [RS:0;f64cf8bdf19b:34437 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T07:40:37,700 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,701 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T07:40:37,707 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T07:40:37,708 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,709 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,709 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,709 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,709 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,709 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,710 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=2, maxPoolSize=2 2024-11-28T07:40:37,710 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,710 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,710 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,710 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,710 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,711 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f64cf8bdf19b:0, corePoolSize=1, maxPoolSize=1 2024-11-28T07:40:37,711 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:40:37,711 DEBUG [RS:0;f64cf8bdf19b:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0, corePoolSize=3, maxPoolSize=3 2024-11-28T07:40:37,712 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,712 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,712 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,713 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,713 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,713 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,34437,1732779635619-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:40:37,734 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T07:40:37,736 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,34437,1732779635619-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,737 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,737 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.Replication(171): f64cf8bdf19b,34437,1732779635619 started 2024-11-28T07:40:37,757 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:37,757 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1482): Serving as f64cf8bdf19b,34437,1732779635619, RpcServer on f64cf8bdf19b/172.17.0.2:34437, sessionid=0x1015d84d6970001 2024-11-28T07:40:37,758 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T07:40:37,758 DEBUG [RS:0;f64cf8bdf19b:34437 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:37,759 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,34437,1732779635619' 2024-11-28T07:40:37,759 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T07:40:37,760 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T07:40:37,761 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T07:40:37,761 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T07:40:37,761 DEBUG [RS:0;f64cf8bdf19b:34437 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:37,761 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f64cf8bdf19b,34437,1732779635619' 2024-11-28T07:40:37,761 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T07:40:37,762 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T07:40:37,762 DEBUG [RS:0;f64cf8bdf19b:34437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T07:40:37,762 INFO [RS:0;f64cf8bdf19b:34437 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T07:40:37,762 INFO [RS:0;f64cf8bdf19b:34437 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T07:40:37,868 INFO [RS:0;f64cf8bdf19b:34437 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T07:40:37,872 INFO [RS:0;f64cf8bdf19b:34437 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f64cf8bdf19b%2C34437%2C1732779635619, suffix=, logDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/WALs/f64cf8bdf19b,34437,1732779635619, archiveDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/oldWALs, maxLogs=32 2024-11-28T07:40:37,889 DEBUG [RS:0;f64cf8bdf19b:34437 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/WALs/f64cf8bdf19b,34437,1732779635619/f64cf8bdf19b%2C34437%2C1732779635619.1732779637874, exclude list is [], retry=0 2024-11-28T07:40:37,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36239,DS-c3cb718e-e593-4a50-82ce-5ce19113defd,DISK] 2024-11-28T07:40:37,898 INFO [RS:0;f64cf8bdf19b:34437 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/WALs/f64cf8bdf19b,34437,1732779635619/f64cf8bdf19b%2C34437%2C1732779635619.1732779637874 2024-11-28T07:40:37,899 DEBUG [RS:0;f64cf8bdf19b:34437 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37517:37517)] 2024-11-28T07:40:37,977 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T07:40:37,978 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:40:37,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741833_1009 (size=32) 2024-11-28T07:40:38,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:40:38,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T07:40:38,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:40:38,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T07:40:38,398 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:40:38,398 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T07:40:38,401 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:40:38,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T07:40:38,404 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:40:38,404 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:40:38,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740 2024-11-28T07:40:38,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740 2024-11-28T07:40:38,410 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:40:38,410 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:40:38,411 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:40:38,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:40:38,417 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:40:38,418 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67165347, jitterRate=8.416622877120972E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:40:38,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732779638390Initializing all the Stores at 1732779638392 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779638392Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779638392Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779638392Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779638392Cleaning up temporary data from old regions at 1732779638410 (+18 ms)Region opened successfully at 1732779638421 (+11 ms) 2024-11-28T07:40:38,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:40:38,421 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:40:38,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:40:38,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:40:38,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:40:38,423 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:40:38,424 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732779638421Disabling compacts and flushes for region at 1732779638421Disabling writes for close at 1732779638422 (+1 ms)Writing region close event to WAL at 1732779638423 (+1 ms)Closed at 1732779638423 2024-11-28T07:40:38,427 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:40:38,427 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T07:40:38,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T07:40:38,442 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:40:38,445 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T07:40:38,598 DEBUG [f64cf8bdf19b:39063 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T07:40:38,611 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:38,618 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,34437,1732779635619, state=OPENING 2024-11-28T07:40:38,623 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T07:40:38,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:38,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:40:38,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:40:38,627 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:40:38,628 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T07:40:38,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:40:38,805 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T07:40:38,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55273, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T07:40:38,820 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T07:40:38,821 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T07:40:38,821 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T07:40:38,825 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f64cf8bdf19b%2C34437%2C1732779635619.meta, suffix=.meta, logDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/WALs/f64cf8bdf19b,34437,1732779635619, archiveDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/oldWALs, maxLogs=32 2024-11-28T07:40:38,843 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/WALs/f64cf8bdf19b,34437,1732779635619/f64cf8bdf19b%2C34437%2C1732779635619.meta.1732779638827.meta, exclude list is [], retry=0 2024-11-28T07:40:38,847 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36239,DS-c3cb718e-e593-4a50-82ce-5ce19113defd,DISK] 2024-11-28T07:40:38,850 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/WALs/f64cf8bdf19b,34437,1732779635619/f64cf8bdf19b%2C34437%2C1732779635619.meta.1732779638827.meta 2024-11-28T07:40:38,850 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37517:37517)] 2024-11-28T07:40:38,851 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:40:38,852 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T07:40:38,855 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T07:40:38,860 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T07:40:38,864 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T07:40:38,865 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:40:38,865 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T07:40:38,865 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T07:40:38,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T07:40:38,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T07:40:38,870 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T07:40:38,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T07:40:38,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T07:40:38,875 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T07:40:38,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T07:40:38,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T07:40:38,878 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:38,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T07:40:38,879 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T07:40:38,880 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740 2024-11-28T07:40:38,882 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740 2024-11-28T07:40:38,885 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T07:40:38,885 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T07:40:38,886 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:40:38,888 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T07:40:38,890 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65242717, jitterRate=-0.027807757258415222}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:40:38,890 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T07:40:38,891 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732779638865Writing region info on filesystem at 1732779638866 (+1 ms)Initializing all the Stores at 1732779638867 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779638867Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779638868 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779638868Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732779638868Cleaning up temporary data from old regions at 1732779638885 (+17 ms)Running coprocessor post-open hooks at 1732779638890 (+5 ms)Region opened successfully at 1732779638891 (+1 ms) 2024-11-28T07:40:38,898 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732779638796 2024-11-28T07:40:38,910 DEBUG [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T07:40:38,911 INFO [RS_OPEN_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T07:40:38,912 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:38,915 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f64cf8bdf19b,34437,1732779635619, state=OPEN 2024-11-28T07:40:38,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:40:38,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T07:40:38,922 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:40:38,922 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T07:40:38,923 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:38,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T07:40:38,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f64cf8bdf19b,34437,1732779635619 in 294 msec 2024-11-28T07:40:38,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T07:40:38,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 496 msec 2024-11-28T07:40:38,937 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T07:40:38,937 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T07:40:38,959 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:38,960 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:38,981 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:38,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:39,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5290 sec 2024-11-28T07:40:39,003 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732779639003, completionTime=-1 2024-11-28T07:40:39,006 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T07:40:39,007 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T07:40:39,061 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T07:40:39,061 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732779699061 2024-11-28T07:40:39,062 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732779759062 2024-11-28T07:40:39,062 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 54 msec 2024-11-28T07:40:39,064 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,39063,1732779634677-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:39,064 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,39063,1732779634677-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:39,064 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,39063,1732779634677-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:39,066 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f64cf8bdf19b:39063, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:39,066 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:39,067 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:39,073 DEBUG [master/f64cf8bdf19b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T07:40:39,093 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.288sec 2024-11-28T07:40:39,094 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T07:40:39,095 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T07:40:39,096 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T07:40:39,097 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T07:40:39,097 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T07:40:39,098 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,39063,1732779634677-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T07:40:39,098 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,39063,1732779634677-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T07:40:39,106 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T07:40:39,106 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T07:40:39,107 INFO [master/f64cf8bdf19b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f64cf8bdf19b,39063,1732779634677-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T07:40:39,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:39,151 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T07:40:39,152 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T07:40:39,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:39,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:39,164 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:39,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a872003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:39,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:39,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:39,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:39,174 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:39,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:39,177 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:39,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:39,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:39,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:39,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:39,218 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=277, ProcessCount=11, AvailableMemoryMB=7597 2024-11-28T07:40:39,226 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:40:39,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:39,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@8dbf8bc 2024-11-28T07:40:39,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:40:39,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:40:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:40:39,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:40:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T07:40:39,254 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:40:39,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-28T07:40:39,256 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:39,258 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:40:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:40:39,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741835_1011 (size=1061) 2024-11-28T07:40:39,271 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:40:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741836_1012 (size=53) 2024-11-28T07:40:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:40:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:40:39,683 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:40:39,683 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e36af693bfc3c757988aba93aff1bdda, disabling compactions & flushes 2024-11-28T07:40:39,683 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:39,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:39,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. after waiting 0 ms 2024-11-28T07:40:39,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:39,684 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:39,684 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e36af693bfc3c757988aba93aff1bdda: Waiting for close lock at 1732779639683Disabling compacts and flushes for region at 1732779639683Disabling writes for close at 1732779639684 (+1 ms)Writing region close event to WAL at 1732779639684Closed at 1732779639684 2024-11-28T07:40:39,686 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:40:39,691 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732779639686"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779639686"}]},"ts":"1732779639686"} 2024-11-28T07:40:39,696 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:40:39,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:40:39,700 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779639698"}]},"ts":"1732779639698"} 2024-11-28T07:40:39,704 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T07:40:39,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, ASSIGN}] 2024-11-28T07:40:39,708 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, ASSIGN 2024-11-28T07:40:39,709 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=false 2024-11-28T07:40:39,861 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e36af693bfc3c757988aba93aff1bdda, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:39,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, ASSIGN because future has completed 2024-11-28T07:40:39,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:40:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:40:40,026 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,026 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:40:40,027 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,027 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:40:40,027 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,027 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,030 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,033 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:40:40,033 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e36af693bfc3c757988aba93aff1bdda columnFamilyName A 2024-11-28T07:40:40,034 DEBUG [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:40,035 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.HStore(327): Store=e36af693bfc3c757988aba93aff1bdda/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:40:40,035 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,037 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:40:40,037 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e36af693bfc3c757988aba93aff1bdda columnFamilyName B 2024-11-28T07:40:40,037 DEBUG [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:40,038 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.HStore(327): Store=e36af693bfc3c757988aba93aff1bdda/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:40:40,038 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,040 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:40:40,041 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e36af693bfc3c757988aba93aff1bdda columnFamilyName C 2024-11-28T07:40:40,041 DEBUG [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:40:40,042 INFO [StoreOpener-e36af693bfc3c757988aba93aff1bdda-1 {}] regionserver.HStore(327): Store=e36af693bfc3c757988aba93aff1bdda/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:40:40,042 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,043 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,044 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,045 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,046 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,046 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,047 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:40:40,049 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,053 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:40:40,054 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e36af693bfc3c757988aba93aff1bdda; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63957732, jitterRate=-0.04695552587509155}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:40:40,054 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,055 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e36af693bfc3c757988aba93aff1bdda: Running coprocessor pre-open hook at 1732779640027Writing region info on filesystem at 1732779640027Initializing all the Stores at 1732779640029 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779640029Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779640029Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779640029Cleaning up temporary data from old regions at 1732779640046 (+17 ms)Running coprocessor post-open hooks at 1732779640054 (+8 ms)Region opened successfully at 1732779640055 (+1 ms) 2024-11-28T07:40:40,057 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., pid=6, masterSystemTime=1732779640019 2024-11-28T07:40:40,061 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,061 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,062 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e36af693bfc3c757988aba93aff1bdda, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:40,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:40:40,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T07:40:40,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 in 202 msec 2024-11-28T07:40:40,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T07:40:40,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, ASSIGN in 366 msec 2024-11-28T07:40:40,078 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:40:40,079 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779640079"}]},"ts":"1732779640079"} 2024-11-28T07:40:40,082 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T07:40:40,083 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:40:40,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 837 msec 2024-11-28T07:40:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T07:40:40,402 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:40,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583d40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,413 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665358ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,422 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:40:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,39063,1732779634677 2024-11-28T07:40:40,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b26fa3 2024-11-28T07:40:40,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:40:40,428 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:40:40,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,433 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,436 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,440 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,443 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,446 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,449 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dac934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,454 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,457 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e341cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,461 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f695ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,463 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b32a20f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,469 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3c1f14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,471 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffa2de8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,474 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec64ade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,476 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e58d725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,480 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c287047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,482 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:40:40,484 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:40:40,485 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:40:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:40:40,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:40:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311f9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:40:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:40:40,486 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:40:40,487 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:40:40,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:40:40,492 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,492 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,492 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,492 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,493 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,494 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,495 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,495 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,495 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,496 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,497 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,499 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,505 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:40:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:40:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:40:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:40:40,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:40:40,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-28T07:40:40,536 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:40,541 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T07:40:40,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:40,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:40:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:40,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:40,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:40,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T07:40:40,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6671735b0b7f438088098027d1a70026 is 50, key is test_row_0/A:col10/1732779640559/Put/seqid=0 2024-11-28T07:40:40,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741837_1013 (size=12001) 2024-11-28T07:40:40,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T07:40:40,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:40,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:40,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:40,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732779640601, max timestamp=1732779640802Num uniques 859; 2024-11-28T07:40:40,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732779640601, max timestamp=1732779640802Num uniques 859; 2024-11-28T07:40:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779700795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:40,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T07:40:40,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T07:40:40,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732779640601, max timestamp=1732779640802Num uniques 859; 2024-11-28T07:40:40,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T07:40:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T07:40:40,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T07:40:40,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:40,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:40,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:40,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:40,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779700873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:40,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779700926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779700926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779700926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:40,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779700961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779701031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T07:40:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779701042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779701042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779701051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:41,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:41,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:41,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6671735b0b7f438088098027d1a70026 2024-11-28T07:40:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T07:40:41,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2f93a950f50f4dd295572776f1f12582 is 50, key is test_row_0/B:col10/1732779640559/Put/seqid=0 2024-11-28T07:40:41,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779701181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741838_1014 (size=12001) 2024-11-28T07:40:41,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2f93a950f50f4dd295572776f1f12582 2024-11-28T07:40:41,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T07:40:41,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:41,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:41,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:41,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779701241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1c3620e9728f4ed89dc2dcc4ee6c8275 is 50, key is test_row_0/C:col10/1732779640559/Put/seqid=0 2024-11-28T07:40:41,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779701251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779701261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779701262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741839_1015 (size=12001) 2024-11-28T07:40:41,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1c3620e9728f4ed89dc2dcc4ee6c8275 2024-11-28T07:40:41,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6671735b0b7f438088098027d1a70026 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6671735b0b7f438088098027d1a70026 2024-11-28T07:40:41,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6671735b0b7f438088098027d1a70026, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T07:40:41,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2f93a950f50f4dd295572776f1f12582 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2f93a950f50f4dd295572776f1f12582 2024-11-28T07:40:41,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2f93a950f50f4dd295572776f1f12582, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T07:40:41,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1c3620e9728f4ed89dc2dcc4ee6c8275 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c3620e9728f4ed89dc2dcc4ee6c8275 2024-11-28T07:40:41,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c3620e9728f4ed89dc2dcc4ee6c8275, entries=150, sequenceid=12, filesize=11.7 K 2024-11-28T07:40:41,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=132.99 KB/136185 for e36af693bfc3c757988aba93aff1bdda in 772ms, sequenceid=12, compaction requested=false 2024-11-28T07:40:41,350 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T07:40:41,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:41,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-28T07:40:41,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,368 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=132.99 KB heapSize=382.24 KB 2024-11-28T07:40:41,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:41,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:41,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/92c7e7aeb3904f2ababab399b45c0709 is 50, key is test_row_0/A:col10/1732779640872/Put/seqid=0 2024-11-28T07:40:41,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741840_1016 (size=12001) 2024-11-28T07:40:41,407 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/92c7e7aeb3904f2ababab399b45c0709 2024-11-28T07:40:41,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/377adeb654ea4e0cb403e6ce6b471d45 is 50, key is test_row_0/B:col10/1732779640872/Put/seqid=0 2024-11-28T07:40:41,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741841_1017 (size=12001) 2024-11-28T07:40:41,456 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/377adeb654ea4e0cb403e6ce6b471d45 2024-11-28T07:40:41,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/8ebedf8350e44067850d4e0d53ca7973 is 50, key is test_row_0/C:col10/1732779640872/Put/seqid=0 2024-11-28T07:40:41,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741842_1018 (size=12001) 2024-11-28T07:40:41,505 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/8ebedf8350e44067850d4e0d53ca7973 2024-11-28T07:40:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:41,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:41,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/92c7e7aeb3904f2ababab399b45c0709 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92c7e7aeb3904f2ababab399b45c0709 2024-11-28T07:40:41,533 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92c7e7aeb3904f2ababab399b45c0709, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T07:40:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/377adeb654ea4e0cb403e6ce6b471d45 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/377adeb654ea4e0cb403e6ce6b471d45 2024-11-28T07:40:41,549 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/377adeb654ea4e0cb403e6ce6b471d45, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T07:40:41,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/8ebedf8350e44067850d4e0d53ca7973 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/8ebedf8350e44067850d4e0d53ca7973 2024-11-28T07:40:41,569 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/8ebedf8350e44067850d4e0d53ca7973, entries=150, sequenceid=49, filesize=11.7 K 2024-11-28T07:40:41,573 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~132.99 KB/136185, heapSize ~382.20 KB/391368, currentSize=20.13 KB/20610 for e36af693bfc3c757988aba93aff1bdda in 203ms, sequenceid=49, compaction requested=false 2024-11-28T07:40:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-28T07:40:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-28T07:40:41,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T07:40:41,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0340 sec 2024-11-28T07:40:41,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0580 sec 2024-11-28T07:40:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:41,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:40:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:41,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:41,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/aca670d2a3414912a4c844c201b0666b is 50, key is test_row_0/A:col10/1732779641570/Put/seqid=0 2024-11-28T07:40:41,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741843_1019 (size=14341) 2024-11-28T07:40:41,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/aca670d2a3414912a4c844c201b0666b 2024-11-28T07:40:41,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/d75901216845467e8da20df1c1a12dac is 50, key is test_row_0/B:col10/1732779641570/Put/seqid=0 2024-11-28T07:40:41,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741844_1020 (size=12001) 2024-11-28T07:40:41,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/d75901216845467e8da20df1c1a12dac 2024-11-28T07:40:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T07:40:41,684 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:41,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-28T07:40:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T07:40:41,690 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:41,692 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:41,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:41,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9a1ddf486daf45898cc121d3bc709a74 is 50, key is test_row_0/C:col10/1732779641570/Put/seqid=0 2024-11-28T07:40:41,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732779641663, max timestamp=1732779641705Num uniques 859; 2024-11-28T07:40:41,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T07:40:41,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732779641663, max timestamp=1732779641705Num uniques 859; 2024-11-28T07:40:41,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-28T07:40:41,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779641682, max timestamp=1732779641705Num uniques 859; 2024-11-28T07:40:41,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:40:41,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741845_1021 (size=12001) 2024-11-28T07:40:41,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9a1ddf486daf45898cc121d3bc709a74 2024-11-28T07:40:41,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/aca670d2a3414912a4c844c201b0666b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aca670d2a3414912a4c844c201b0666b 2024-11-28T07:40:41,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aca670d2a3414912a4c844c201b0666b, entries=200, sequenceid=59, filesize=14.0 K 2024-11-28T07:40:41,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/d75901216845467e8da20df1c1a12dac as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/d75901216845467e8da20df1c1a12dac 2024-11-28T07:40:41,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/d75901216845467e8da20df1c1a12dac, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T07:40:41,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9a1ddf486daf45898cc121d3bc709a74 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9a1ddf486daf45898cc121d3bc709a74 2024-11-28T07:40:41,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779701762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779701762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779701766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779701769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779701771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T07:40:41,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9a1ddf486daf45898cc121d3bc709a74, entries=150, sequenceid=59, filesize=11.7 K 2024-11-28T07:40:41,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=138.65 KB/141980 for e36af693bfc3c757988aba93aff1bdda in 198ms, sequenceid=59, compaction requested=true 2024-11-28T07:40:41,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:41,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:41,807 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:41,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:41,810 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:41,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:41,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:41,813 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:41,814 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:41,815 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,815 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2f93a950f50f4dd295572776f1f12582, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/377adeb654ea4e0cb403e6ce6b471d45, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/d75901216845467e8da20df1c1a12dac] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=35.2 K 2024-11-28T07:40:41,816 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:41,816 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:41,816 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,817 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f93a950f50f4dd295572776f1f12582, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732779640520 2024-11-28T07:40:41,817 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6671735b0b7f438088098027d1a70026, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92c7e7aeb3904f2ababab399b45c0709, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aca670d2a3414912a4c844c201b0666b] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.4 K 2024-11-28T07:40:41,818 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6671735b0b7f438088098027d1a70026, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732779640520 2024-11-28T07:40:41,818 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 377adeb654ea4e0cb403e6ce6b471d45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732779640858 2024-11-28T07:40:41,819 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d75901216845467e8da20df1c1a12dac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732779641570 2024-11-28T07:40:41,819 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c7e7aeb3904f2ababab399b45c0709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732779640858 2024-11-28T07:40:41,820 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting aca670d2a3414912a4c844c201b0666b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732779641565 2024-11-28T07:40:41,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-28T07:40:41,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,848 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=138.65 KB heapSize=385.77 KB 2024-11-28T07:40:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:41,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:41,864 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:41,865 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/e4f236af8ea040308fbad3f6a78ac154 is 50, key is test_row_0/B:col10/1732779641570/Put/seqid=0 2024-11-28T07:40:41,874 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:41,875 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/e3ab1c0aec7d42149d8cafc28c240031 is 50, key is test_row_0/A:col10/1732779641570/Put/seqid=0 2024-11-28T07:40:41,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/92b2ee598c1b45c1824ed6de46f8a0ec is 50, key is test_row_0/A:col10/1732779641765/Put/seqid=0 2024-11-28T07:40:41,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741846_1022 (size=12104) 2024-11-28T07:40:41,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741847_1023 (size=12104) 2024-11-28T07:40:41,904 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/e3ab1c0aec7d42149d8cafc28c240031 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e3ab1c0aec7d42149d8cafc28c240031 2024-11-28T07:40:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:41,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:41,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741848_1024 (size=12001) 2024-11-28T07:40:41,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779701926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779701930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779701930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,940 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into e3ab1c0aec7d42149d8cafc28c240031(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:41,941 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,943 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779641804; duration=0sec 2024-11-28T07:40:41,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,944 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:41,944 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779701934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,944 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779701938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:41,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:41,947 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:41,947 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:41,947 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:41,947 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c3620e9728f4ed89dc2dcc4ee6c8275, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/8ebedf8350e44067850d4e0d53ca7973, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9a1ddf486daf45898cc121d3bc709a74] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=35.2 K 2024-11-28T07:40:41,948 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c3620e9728f4ed89dc2dcc4ee6c8275, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732779640520 2024-11-28T07:40:41,949 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ebedf8350e44067850d4e0d53ca7973, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732779640858 2024-11-28T07:40:41,950 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a1ddf486daf45898cc121d3bc709a74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732779641570 2024-11-28T07:40:41,973 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:41,974 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/511ab0dffeb44e16987574ffe8c76674 is 50, key is test_row_0/C:col10/1732779641570/Put/seqid=0 2024-11-28T07:40:41,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741849_1025 (size=12104) 2024-11-28T07:40:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T07:40:42,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779702052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779702052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779702052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779702053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779702053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779702261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779702262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779702262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779702272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779702272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,302 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/e4f236af8ea040308fbad3f6a78ac154 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/e4f236af8ea040308fbad3f6a78ac154 2024-11-28T07:40:42,317 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into e4f236af8ea040308fbad3f6a78ac154(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:42,317 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:42,318 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779641810; duration=0sec 2024-11-28T07:40:42,318 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:42,318 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:42,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T07:40:42,325 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/92b2ee598c1b45c1824ed6de46f8a0ec 2024-11-28T07:40:42,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/24c2ba3d63534804bcc3d726c300a57f is 50, key is test_row_0/B:col10/1732779641765/Put/seqid=0 2024-11-28T07:40:42,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741850_1026 (size=12001) 2024-11-28T07:40:42,375 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/24c2ba3d63534804bcc3d726c300a57f 2024-11-28T07:40:42,411 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/511ab0dffeb44e16987574ffe8c76674 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/511ab0dffeb44e16987574ffe8c76674 2024-11-28T07:40:42,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b82be6c773c348edb121d3b52f4d39c3 is 50, key is test_row_0/C:col10/1732779641765/Put/seqid=0 2024-11-28T07:40:42,435 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 511ab0dffeb44e16987574ffe8c76674(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:42,436 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:42,436 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779641810; duration=0sec 2024-11-28T07:40:42,436 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:42,436 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:42,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741851_1027 (size=12001) 2024-11-28T07:40:42,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b82be6c773c348edb121d3b52f4d39c3 2024-11-28T07:40:42,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/92b2ee598c1b45c1824ed6de46f8a0ec as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92b2ee598c1b45c1824ed6de46f8a0ec 2024-11-28T07:40:42,502 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92b2ee598c1b45c1824ed6de46f8a0ec, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T07:40:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/24c2ba3d63534804bcc3d726c300a57f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/24c2ba3d63534804bcc3d726c300a57f 2024-11-28T07:40:42,515 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/24c2ba3d63534804bcc3d726c300a57f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T07:40:42,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b82be6c773c348edb121d3b52f4d39c3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b82be6c773c348edb121d3b52f4d39c3 2024-11-28T07:40:42,531 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b82be6c773c348edb121d3b52f4d39c3, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T07:40:42,533 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~385.72 KB/394976, currentSize=46.96 KB/48090 for e36af693bfc3c757988aba93aff1bdda in 685ms, sequenceid=94, compaction requested=false 2024-11-28T07:40:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-28T07:40:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-28T07:40:42,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-28T07:40:42,545 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 846 msec 2024-11-28T07:40:42,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 862 msec 2024-11-28T07:40:42,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:40:42,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:42,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:42,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:42,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:42,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:42,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:42,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/bb1ee655fca04fe9ab18c7d9a654ee18 is 50, key is test_row_0/A:col10/1732779641929/Put/seqid=0 2024-11-28T07:40:42,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741852_1028 (size=12001) 2024-11-28T07:40:42,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/bb1ee655fca04fe9ab18c7d9a654ee18 2024-11-28T07:40:42,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779702627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779702629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779702643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c2b1cca29e3346f2b72858011348074c is 50, key is test_row_0/B:col10/1732779641929/Put/seqid=0 2024-11-28T07:40:42,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779702649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779702649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741853_1029 (size=12001) 2024-11-28T07:40:42,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c2b1cca29e3346f2b72858011348074c 2024-11-28T07:40:42,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/51057362353e49c4ae19b6fbce700e43 is 50, key is test_row_0/C:col10/1732779641929/Put/seqid=0 2024-11-28T07:40:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779702760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779702761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779702761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779702762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779702762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:42,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741854_1030 (size=12001) 2024-11-28T07:40:42,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/51057362353e49c4ae19b6fbce700e43 2024-11-28T07:40:42,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/bb1ee655fca04fe9ab18c7d9a654ee18 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/bb1ee655fca04fe9ab18c7d9a654ee18 2024-11-28T07:40:42,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/bb1ee655fca04fe9ab18c7d9a654ee18, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T07:40:42,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c2b1cca29e3346f2b72858011348074c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c2b1cca29e3346f2b72858011348074c 2024-11-28T07:40:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-28T07:40:42,833 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:42,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-28T07:40:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T07:40:42,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:42,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:42,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:42,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c2b1cca29e3346f2b72858011348074c, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T07:40:42,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/51057362353e49c4ae19b6fbce700e43 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/51057362353e49c4ae19b6fbce700e43 2024-11-28T07:40:42,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/51057362353e49c4ae19b6fbce700e43, entries=150, sequenceid=109, filesize=11.7 K 2024-11-28T07:40:42,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e36af693bfc3c757988aba93aff1bdda in 295ms, sequenceid=109, compaction requested=true 2024-11-28T07:40:42,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:42,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:42,873 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:42,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:42,873 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:42,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:42,875 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:42,876 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:42,876 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:42,876 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e3ab1c0aec7d42149d8cafc28c240031, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92b2ee598c1b45c1824ed6de46f8a0ec, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/bb1ee655fca04fe9ab18c7d9a654ee18] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=35.3 K 2024-11-28T07:40:42,876 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:42,877 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:42,877 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ab1c0aec7d42149d8cafc28c240031, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732779641570 2024-11-28T07:40:42,877 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:42,877 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/e4f236af8ea040308fbad3f6a78ac154, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/24c2ba3d63534804bcc3d726c300a57f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c2b1cca29e3346f2b72858011348074c] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=35.3 K 2024-11-28T07:40:42,878 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92b2ee598c1b45c1824ed6de46f8a0ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779641757 2024-11-28T07:40:42,878 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e4f236af8ea040308fbad3f6a78ac154, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732779641570 2024-11-28T07:40:42,879 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c2ba3d63534804bcc3d726c300a57f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779641757 2024-11-28T07:40:42,879 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1ee655fca04fe9ab18c7d9a654ee18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732779641923 2024-11-28T07:40:42,879 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b1cca29e3346f2b72858011348074c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732779641923 2024-11-28T07:40:42,908 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:42,909 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/dc38d681eaa245c98e00b04ed3c9c776 is 50, key is test_row_0/A:col10/1732779641929/Put/seqid=0 2024-11-28T07:40:42,909 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:42,910 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/9a0c51aa58cc4e9d8e52389d8dd75ece is 50, key is test_row_0/B:col10/1732779641929/Put/seqid=0 2024-11-28T07:40:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T07:40:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741855_1031 (size=12207) 2024-11-28T07:40:42,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741856_1032 (size=12207) 2024-11-28T07:40:42,978 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/dc38d681eaa245c98e00b04ed3c9c776 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc38d681eaa245c98e00b04ed3c9c776 2024-11-28T07:40:42,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779642622, max timestamp=1732779642983Num uniques 859; 2024-11-28T07:40:42,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:40:42,990 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into dc38d681eaa245c98e00b04ed3c9c776(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:42,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:42,991 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779642873; duration=0sec 2024-11-28T07:40:42,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:42,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:42,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:42,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=96.16 KB heapSize=282.03 KB 2024-11-28T07:40:42,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:42,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:42,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:42,993 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:42,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:42,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:42,994 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:42,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:42,994 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:42,994 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/511ab0dffeb44e16987574ffe8c76674, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b82be6c773c348edb121d3b52f4d39c3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/51057362353e49c4ae19b6fbce700e43] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=35.3 K 2024-11-28T07:40:42,995 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 511ab0dffeb44e16987574ffe8c76674, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732779641570 2024-11-28T07:40:42,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T07:40:42,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:42,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:42,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:42,995 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting b82be6c773c348edb121d3b52f4d39c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779641757 2024-11-28T07:40:42,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:42,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:42,996 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51057362353e49c4ae19b6fbce700e43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732779641923 2024-11-28T07:40:42,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:43,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/e594158a801145abace8bd2122f13e83 is 50, key is test_row_0/A:col10/1732779642977/Put/seqid=0 2024-11-28T07:40:43,023 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:43,024 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1c200b99b0764ac88afbd047acbcb7d0 is 50, key is test_row_0/C:col10/1732779641929/Put/seqid=0 2024-11-28T07:40:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741857_1033 (size=16781) 2024-11-28T07:40:43,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/e594158a801145abace8bd2122f13e83 2024-11-28T07:40:43,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779703038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779703038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779703041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779703043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779703058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741858_1034 (size=12207) 2024-11-28T07:40:43,085 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1c200b99b0764ac88afbd047acbcb7d0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c200b99b0764ac88afbd047acbcb7d0 2024-11-28T07:40:43,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/608339954b274b88bfdf527a0459cb01 is 50, key is test_row_0/B:col10/1732779642977/Put/seqid=0 2024-11-28T07:40:43,101 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 1c200b99b0764ac88afbd047acbcb7d0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:43,101 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:43,102 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779642874; duration=0sec 2024-11-28T07:40:43,102 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:43,102 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741859_1035 (size=12101) 2024-11-28T07:40:43,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/608339954b274b88bfdf527a0459cb01 2024-11-28T07:40:43,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T07:40:43,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T07:40:43,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:43,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:43,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:43,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/628f2710c8044e1fbda00950679a943a is 50, key is test_row_0/C:col10/1732779642977/Put/seqid=0 2024-11-28T07:40:43,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779703163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779703163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779703163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779703164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779703171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741860_1036 (size=12101) 2024-11-28T07:40:43,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/628f2710c8044e1fbda00950679a943a 2024-11-28T07:40:43,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/e594158a801145abace8bd2122f13e83 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e594158a801145abace8bd2122f13e83 2024-11-28T07:40:43,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e594158a801145abace8bd2122f13e83, entries=250, sequenceid=132, filesize=16.4 K 2024-11-28T07:40:43,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/608339954b274b88bfdf527a0459cb01 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/608339954b274b88bfdf527a0459cb01 2024-11-28T07:40:43,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/608339954b274b88bfdf527a0459cb01, entries=150, sequenceid=132, filesize=11.8 K 2024-11-28T07:40:43,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/628f2710c8044e1fbda00950679a943a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/628f2710c8044e1fbda00950679a943a 2024-11-28T07:40:43,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/628f2710c8044e1fbda00950679a943a, entries=150, sequenceid=132, filesize=11.8 K 2024-11-28T07:40:43,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~281.98 KB/288752, currentSize=80.51 KB/82440 for e36af693bfc3c757988aba93aff1bdda in 288ms, sequenceid=132, compaction requested=false 2024-11-28T07:40:43,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:43,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-28T07:40:43,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:43,315 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:40:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:43,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:43,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/19d2288ec9ff407581737f591e731276 is 50, key is test_row_0/A:col10/1732779643048/Put/seqid=0 2024-11-28T07:40:43,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741861_1037 (size=12151) 2024-11-28T07:40:43,362 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/19d2288ec9ff407581737f591e731276 2024-11-28T07:40:43,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/20e0b868d59e4b6e8181d1e49d6d5341 is 50, key is test_row_0/B:col10/1732779643048/Put/seqid=0 2024-11-28T07:40:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:43,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:43,396 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/9a0c51aa58cc4e9d8e52389d8dd75ece as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/9a0c51aa58cc4e9d8e52389d8dd75ece 2024-11-28T07:40:43,409 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 9a0c51aa58cc4e9d8e52389d8dd75ece(size=11.9 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:43,409 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:43,409 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779642873; duration=0sec 2024-11-28T07:40:43,409 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:43,409 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:43,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741862_1038 (size=12151) 2024-11-28T07:40:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779703442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779703442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779703443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,455 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779703446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779703446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T07:40:43,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779703561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779703561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779703561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779703561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779703561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,676 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T07:40:43,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779703771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779703780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779703781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:43,821 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/20e0b868d59e4b6e8181d1e49d6d5341 2024-11-28T07:40:43,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/2b6d3c30039f4f59a33d81fb1e9a47a9 is 50, key is test_row_0/C:col10/1732779643048/Put/seqid=0 2024-11-28T07:40:43,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741863_1039 (size=12151) 2024-11-28T07:40:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T07:40:44,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779704091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779704091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,095 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779704091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779704091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779704092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,279 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/2b6d3c30039f4f59a33d81fb1e9a47a9 2024-11-28T07:40:44,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/19d2288ec9ff407581737f591e731276 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/19d2288ec9ff407581737f591e731276 2024-11-28T07:40:44,307 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/19d2288ec9ff407581737f591e731276, entries=150, sequenceid=148, filesize=11.9 K 2024-11-28T07:40:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/20e0b868d59e4b6e8181d1e49d6d5341 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/20e0b868d59e4b6e8181d1e49d6d5341 2024-11-28T07:40:44,319 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/20e0b868d59e4b6e8181d1e49d6d5341, entries=150, sequenceid=148, filesize=11.9 K 2024-11-28T07:40:44,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/2b6d3c30039f4f59a33d81fb1e9a47a9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2b6d3c30039f4f59a33d81fb1e9a47a9 2024-11-28T07:40:44,346 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2b6d3c30039f4f59a33d81fb1e9a47a9, entries=150, sequenceid=148, filesize=11.9 K 2024-11-28T07:40:44,349 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e36af693bfc3c757988aba93aff1bdda in 1035ms, sequenceid=148, compaction requested=true 2024-11-28T07:40:44,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:44,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:44,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-28T07:40:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-28T07:40:44,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T07:40:44,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5120 sec 2024-11-28T07:40:44,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.5250 sec 2024-11-28T07:40:44,390 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:40:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:44,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779643437, max timestamp=1732779644648Num uniques 859; 2024-11-28T07:40:44,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779643437, max timestamp=1732779644648Num uniques 859; 2024-11-28T07:40:44,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:40:44,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:40:44,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732779643437, max timestamp=1732779644648Num uniques 859; 2024-11-28T07:40:44,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-28T07:40:44,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=35.39 KB heapSize=85.63 KB 2024-11-28T07:40:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:44,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6d334e6c0b57493db21d1b7838374dc5 is 50, key is test_row_0/A:col10/1732779643441/Put/seqid=0 2024-11-28T07:40:44,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741864_1040 (size=12151) 2024-11-28T07:40:44,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6d334e6c0b57493db21d1b7838374dc5 2024-11-28T07:40:44,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0096babbe7c144629d8e253c4e57738f is 50, key is test_row_0/B:col10/1732779643441/Put/seqid=0 2024-11-28T07:40:44,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779644736, max timestamp=1732779644765Num uniques 859; 2024-11-28T07:40:44,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:40:44,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779644736, max timestamp=1732779644765Num uniques 859; 2024-11-28T07:40:44,772 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:40:44,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779644736, max timestamp=1732779644765Num uniques 859; 2024-11-28T07:40:44,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:40:44,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741865_1041 (size=12151) 2024-11-28T07:40:44,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779704820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779704823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779704827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779704827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779704829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779704940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779704940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779704950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779704950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779704950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:44,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-28T07:40:44,981 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:44,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-28T07:40:44,985 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:44,986 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:44,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T07:40:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T07:40:45,114 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T07:40:45,114 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T07:40:45,116 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T07:40:45,116 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T07:40:45,116 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T07:40:45,116 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T07:40:45,118 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T07:40:45,118 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T07:40:45,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T07:40:45,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779705151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779705160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779705160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779705161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779705161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0096babbe7c144629d8e253c4e57738f 2024-11-28T07:40:45,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/a39c254862474d019d1b8e49f46d0d69 is 50, key is test_row_0/C:col10/1732779643441/Put/seqid=0 2024-11-28T07:40:45,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741866_1042 (size=12151) 2024-11-28T07:40:45,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T07:40:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T07:40:45,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T07:40:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:45,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779705461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779705470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779705470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779705470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779705481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T07:40:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T07:40:45,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/a39c254862474d019d1b8e49f46d0d69 2024-11-28T07:40:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6d334e6c0b57493db21d1b7838374dc5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d334e6c0b57493db21d1b7838374dc5 2024-11-28T07:40:45,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d334e6c0b57493db21d1b7838374dc5, entries=150, sequenceid=172, filesize=11.9 K 2024-11-28T07:40:45,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0096babbe7c144629d8e253c4e57738f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0096babbe7c144629d8e253c4e57738f 2024-11-28T07:40:45,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0096babbe7c144629d8e253c4e57738f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-28T07:40:45,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/a39c254862474d019d1b8e49f46d0d69 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a39c254862474d019d1b8e49f46d0d69 2024-11-28T07:40:45,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a39c254862474d019d1b8e49f46d0d69, entries=150, sequenceid=172, filesize=11.9 K 2024-11-28T07:40:45,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~48.80 KB/49976, heapSize ~125.43 KB/128440, currentSize=140.89 KB/144270 for e36af693bfc3c757988aba93aff1bdda in 1035ms, sequenceid=172, compaction requested=true 2024-11-28T07:40:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:45,699 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:40:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:45,699 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:40:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:45,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:45,702 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:40:45,702 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:40:45,703 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:45,703 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:45,703 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,703 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,703 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc38d681eaa245c98e00b04ed3c9c776, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e594158a801145abace8bd2122f13e83, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/19d2288ec9ff407581737f591e731276, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d334e6c0b57493db21d1b7838374dc5] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=52.0 K 2024-11-28T07:40:45,703 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/9a0c51aa58cc4e9d8e52389d8dd75ece, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/608339954b274b88bfdf527a0459cb01, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/20e0b868d59e4b6e8181d1e49d6d5341, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0096babbe7c144629d8e253c4e57738f] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=47.5 K 2024-11-28T07:40:45,703 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc38d681eaa245c98e00b04ed3c9c776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732779641923 2024-11-28T07:40:45,703 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0c51aa58cc4e9d8e52389d8dd75ece, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732779641923 2024-11-28T07:40:45,704 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e594158a801145abace8bd2122f13e83, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732779642622 2024-11-28T07:40:45,704 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 608339954b274b88bfdf527a0459cb01, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732779642645 2024-11-28T07:40:45,705 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d2288ec9ff407581737f591e731276, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732779643025 2024-11-28T07:40:45,705 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e0b868d59e4b6e8181d1e49d6d5341, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732779643025 2024-11-28T07:40:45,705 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d334e6c0b57493db21d1b7838374dc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732779643441 2024-11-28T07:40:45,706 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0096babbe7c144629d8e253c4e57738f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732779643441 2024-11-28T07:40:45,726 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:45,727 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/be6db0b9d16c48fbba64f07b98007790 is 50, key is test_row_0/A:col10/1732779643441/Put/seqid=0 2024-11-28T07:40:45,730 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:45,731 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2cb613b9dc8e428daa3917b9e2641f55 is 50, key is test_row_0/B:col10/1732779643441/Put/seqid=0 2024-11-28T07:40:45,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741868_1044 (size=12493) 2024-11-28T07:40:45,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-28T07:40:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,762 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-28T07:40:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:45,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:45,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741867_1043 (size=12493) 2024-11-28T07:40:45,779 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2cb613b9dc8e428daa3917b9e2641f55 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2cb613b9dc8e428daa3917b9e2641f55 2024-11-28T07:40:45,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/40a3547224aa4d358fef76fe14569ea2 is 50, key is test_row_0/A:col10/1732779644819/Put/seqid=0 2024-11-28T07:40:45,788 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/be6db0b9d16c48fbba64f07b98007790 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/be6db0b9d16c48fbba64f07b98007790 2024-11-28T07:40:45,790 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 2cb613b9dc8e428daa3917b9e2641f55(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:45,790 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:45,791 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=12, startTime=1732779645699; duration=0sec 2024-11-28T07:40:45,791 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:45,791 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:45,791 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:40:45,794 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:40:45,794 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:45,794 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:45,794 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c200b99b0764ac88afbd047acbcb7d0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/628f2710c8044e1fbda00950679a943a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2b6d3c30039f4f59a33d81fb1e9a47a9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a39c254862474d019d1b8e49f46d0d69] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=47.5 K 2024-11-28T07:40:45,795 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c200b99b0764ac88afbd047acbcb7d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1732779641923 2024-11-28T07:40:45,796 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 628f2710c8044e1fbda00950679a943a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732779642645 2024-11-28T07:40:45,797 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b6d3c30039f4f59a33d81fb1e9a47a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732779643025 2024-11-28T07:40:45,801 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into be6db0b9d16c48fbba64f07b98007790(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:45,801 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:45,801 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a39c254862474d019d1b8e49f46d0d69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732779643441 2024-11-28T07:40:45,801 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=12, startTime=1732779645699; duration=0sec 2024-11-28T07:40:45,801 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:45,801 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:45,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741869_1045 (size=12151) 2024-11-28T07:40:45,813 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/40a3547224aa4d358fef76fe14569ea2 2024-11-28T07:40:45,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/dcb75d8c72df4587ab53703dda651d0a is 50, key is test_row_0/B:col10/1732779644819/Put/seqid=0 2024-11-28T07:40:45,845 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:45,846 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/f82dbf86c9994880ad9c378a87236852 is 50, key is test_row_0/C:col10/1732779643441/Put/seqid=0 2024-11-28T07:40:45,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741870_1046 (size=12151) 2024-11-28T07:40:45,865 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/dcb75d8c72df4587ab53703dda651d0a 2024-11-28T07:40:45,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741871_1047 (size=12493) 2024-11-28T07:40:45,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/80a8ebf711d249689e74bf5752a13d33 is 50, key is test_row_0/C:col10/1732779644819/Put/seqid=0 2024-11-28T07:40:45,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741872_1048 (size=12151) 2024-11-28T07:40:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:45,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779705990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779705990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779705991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:45,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779705993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779705994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779706100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779706110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779706110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T07:40:46,285 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/f82dbf86c9994880ad9c378a87236852 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/f82dbf86c9994880ad9c378a87236852 2024-11-28T07:40:46,294 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into f82dbf86c9994880ad9c378a87236852(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:46,294 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:46,294 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=12, startTime=1732779645699; duration=0sec 2024-11-28T07:40:46,294 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:46,294 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:46,299 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/80a8ebf711d249689e74bf5752a13d33 2024-11-28T07:40:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/40a3547224aa4d358fef76fe14569ea2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/40a3547224aa4d358fef76fe14569ea2 2024-11-28T07:40:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779706310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,318 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/40a3547224aa4d358fef76fe14569ea2, entries=150, sequenceid=208, filesize=11.9 K 2024-11-28T07:40:46,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/dcb75d8c72df4587ab53703dda651d0a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/dcb75d8c72df4587ab53703dda651d0a 2024-11-28T07:40:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779706321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779706321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,329 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/dcb75d8c72df4587ab53703dda651d0a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-28T07:40:46,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/80a8ebf711d249689e74bf5752a13d33 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/80a8ebf711d249689e74bf5752a13d33 2024-11-28T07:40:46,344 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/80a8ebf711d249689e74bf5752a13d33, entries=150, sequenceid=208, filesize=11.9 K 2024-11-28T07:40:46,346 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for e36af693bfc3c757988aba93aff1bdda in 584ms, sequenceid=208, compaction requested=false 2024-11-28T07:40:46,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:46,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:46,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-28T07:40:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-28T07:40:46,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T07:40:46,352 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3620 sec 2024-11-28T07:40:46,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3730 sec 2024-11-28T07:40:46,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:46,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:40:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:46,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:46,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/a82716fe10c443c791b17fabbf5e6bbd is 50, key is test_row_0/A:col10/1732779646622/Put/seqid=0 2024-11-28T07:40:46,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741873_1049 (size=14541) 2024-11-28T07:40:46,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/a82716fe10c443c791b17fabbf5e6bbd 2024-11-28T07:40:46,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/065436bee16d47fa8f0347f580ef63e9 is 50, key is test_row_0/B:col10/1732779646622/Put/seqid=0 2024-11-28T07:40:46,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741874_1050 (size=12151) 2024-11-28T07:40:46,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732779646634, max timestamp=1732779646704Num uniques 859; 2024-11-28T07:40:46,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732779646634, max timestamp=1732779646699Num uniques 859; 2024-11-28T07:40:46,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732779646634, max timestamp=1732779646704Num uniques 859; 2024-11-28T07:40:46,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T07:40:46,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T07:40:46,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T07:40:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779706753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779706753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779706757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779706870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779706870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779706870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779707001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779707000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,007 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in e36af693bfc3c757988aba93aff1bdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:40:47,007 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in e36af693bfc3c757988aba93aff1bdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:40:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779707081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779707081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779707081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/065436bee16d47fa8f0347f580ef63e9 2024-11-28T07:40:47,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9be585f57ff44111ac354a48d89f424f is 50, key is test_row_0/C:col10/1732779646622/Put/seqid=0 2024-11-28T07:40:47,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741875_1051 (size=12151) 2024-11-28T07:40:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T07:40:47,131 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:47,132 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-28T07:40:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T07:40:47,135 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:47,136 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:47,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T07:40:47,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T07:40:47,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:47,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779707390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779707391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779707391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T07:40:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:47,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:47,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T07:40:47,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9be585f57ff44111ac354a48d89f424f 2024-11-28T07:40:47,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/a82716fe10c443c791b17fabbf5e6bbd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/a82716fe10c443c791b17fabbf5e6bbd 2024-11-28T07:40:47,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/a82716fe10c443c791b17fabbf5e6bbd, entries=200, sequenceid=221, filesize=14.2 K 2024-11-28T07:40:47,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/065436bee16d47fa8f0347f580ef63e9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/065436bee16d47fa8f0347f580ef63e9 2024-11-28T07:40:47,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/065436bee16d47fa8f0347f580ef63e9, entries=150, sequenceid=221, filesize=11.9 K 2024-11-28T07:40:47,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9be585f57ff44111ac354a48d89f424f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9be585f57ff44111ac354a48d89f424f 2024-11-28T07:40:47,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9be585f57ff44111ac354a48d89f424f, entries=150, sequenceid=221, filesize=11.9 K 2024-11-28T07:40:47,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for e36af693bfc3c757988aba93aff1bdda in 927ms, sequenceid=221, compaction requested=true 2024-11-28T07:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:47,554 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:47,554 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:47,555 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:47,556 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:47,556 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,556 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2cb613b9dc8e428daa3917b9e2641f55, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/dcb75d8c72df4587ab53703dda651d0a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/065436bee16d47fa8f0347f580ef63e9] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=35.9 K 2024-11-28T07:40:47,556 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:47,556 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:47,557 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,557 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/be6db0b9d16c48fbba64f07b98007790, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/40a3547224aa4d358fef76fe14569ea2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/a82716fe10c443c791b17fabbf5e6bbd] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=38.3 K 2024-11-28T07:40:47,558 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb613b9dc8e428daa3917b9e2641f55, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732779643441 2024-11-28T07:40:47,558 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting be6db0b9d16c48fbba64f07b98007790, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732779643441 2024-11-28T07:40:47,558 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb75d8c72df4587ab53703dda651d0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732779644819 2024-11-28T07:40:47,558 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40a3547224aa4d358fef76fe14569ea2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732779644819 2024-11-28T07:40:47,559 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 065436bee16d47fa8f0347f580ef63e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732779645988 2024-11-28T07:40:47,559 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting a82716fe10c443c791b17fabbf5e6bbd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732779645988 2024-11-28T07:40:47,574 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:47,575 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/43b4da80b915459096484433d0ccd914 is 50, key is test_row_0/A:col10/1732779646622/Put/seqid=0 2024-11-28T07:40:47,579 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:47,580 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/4793dc2e24ef406daaf8140a79dce23e is 50, key is test_row_0/B:col10/1732779646622/Put/seqid=0 2024-11-28T07:40:47,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741876_1052 (size=12595) 2024-11-28T07:40:47,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741877_1053 (size=12595) 2024-11-28T07:40:47,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-28T07:40:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,603 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-28T07:40:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:47,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:47,608 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/4793dc2e24ef406daaf8140a79dce23e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4793dc2e24ef406daaf8140a79dce23e 2024-11-28T07:40:47,611 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/43b4da80b915459096484433d0ccd914 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43b4da80b915459096484433d0ccd914 2024-11-28T07:40:47,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/f9f728acb5f04d6386c6a89b4dd58432 is 50, key is test_row_0/A:col10/1732779646748/Put/seqid=0 2024-11-28T07:40:47,622 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 4793dc2e24ef406daaf8140a79dce23e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:47,622 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:47,622 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779647554; duration=0sec 2024-11-28T07:40:47,622 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:47,622 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:47,622 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:47,624 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:47,624 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:47,624 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:47,624 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/f82dbf86c9994880ad9c378a87236852, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/80a8ebf711d249689e74bf5752a13d33, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9be585f57ff44111ac354a48d89f424f] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=35.9 K 2024-11-28T07:40:47,625 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f82dbf86c9994880ad9c378a87236852, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732779643441 2024-11-28T07:40:47,626 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a8ebf711d249689e74bf5752a13d33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732779644819 2024-11-28T07:40:47,628 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be585f57ff44111ac354a48d89f424f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732779645988 2024-11-28T07:40:47,628 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into 43b4da80b915459096484433d0ccd914(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:47,629 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:47,629 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779647554; duration=0sec 2024-11-28T07:40:47,629 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:47,629 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:47,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741878_1054 (size=12151) 2024-11-28T07:40:47,655 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:47,656 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/3c2c619660634a99bd2804e34b1c7538 is 50, key is test_row_0/C:col10/1732779646622/Put/seqid=0 2024-11-28T07:40:47,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741879_1055 (size=12595) 2024-11-28T07:40:47,675 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/3c2c619660634a99bd2804e34b1c7538 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3c2c619660634a99bd2804e34b1c7538 2024-11-28T07:40:47,689 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 3c2c619660634a99bd2804e34b1c7538(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:47,689 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:47,689 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779647554; duration=0sec 2024-11-28T07:40:47,689 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:47,689 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T07:40:47,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:47,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779707914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,921 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779707917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779707920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:47,930 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779708030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,040 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/f9f728acb5f04d6386c6a89b4dd58432 2024-11-28T07:40:48,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779708040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779708041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/ca0da42f8fc44575a6d5bc5c1f597eed is 50, key is test_row_0/B:col10/1732779646748/Put/seqid=0 2024-11-28T07:40:48,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741880_1056 (size=12151) 2024-11-28T07:40:48,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779708240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779708261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779708261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T07:40:48,456 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/ca0da42f8fc44575a6d5bc5c1f597eed 2024-11-28T07:40:48,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1b08af4b5f2c46c5a4853165b326bd3d is 50, key is test_row_0/C:col10/1732779646748/Put/seqid=0 2024-11-28T07:40:48,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741881_1057 (size=12151) 2024-11-28T07:40:48,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779708550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779708571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779708571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:48,879 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1b08af4b5f2c46c5a4853165b326bd3d 2024-11-28T07:40:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/f9f728acb5f04d6386c6a89b4dd58432 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/f9f728acb5f04d6386c6a89b4dd58432 2024-11-28T07:40:48,895 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/f9f728acb5f04d6386c6a89b4dd58432, entries=150, sequenceid=256, filesize=11.9 K 2024-11-28T07:40:48,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/ca0da42f8fc44575a6d5bc5c1f597eed as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/ca0da42f8fc44575a6d5bc5c1f597eed 2024-11-28T07:40:48,903 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/ca0da42f8fc44575a6d5bc5c1f597eed, entries=150, sequenceid=256, filesize=11.9 K 2024-11-28T07:40:48,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1b08af4b5f2c46c5a4853165b326bd3d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1b08af4b5f2c46c5a4853165b326bd3d 2024-11-28T07:40:48,913 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1b08af4b5f2c46c5a4853165b326bd3d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-28T07:40:48,915 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for e36af693bfc3c757988aba93aff1bdda in 1311ms, sequenceid=256, compaction requested=false 2024-11-28T07:40:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:48,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-28T07:40:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-28T07:40:48,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-28T07:40:48,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-11-28T07:40:48,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.7900 sec 2024-11-28T07:40:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:49,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:40:49,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:49,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:49,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6d263b57751c430489bf8696c66f2541 is 50, key is test_row_0/A:col10/1732779647901/Put/seqid=0 2024-11-28T07:40:49,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741882_1058 (size=12301) 2024-11-28T07:40:49,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779709076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779709076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779649066, max timestamp=1732779649075Num uniques 859; 2024-11-28T07:40:49,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:40:49,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779649066, max timestamp=1732779649075Num uniques 859; 2024-11-28T07:40:49,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:40:49,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779649066, max timestamp=1732779649075Num uniques 859; 2024-11-28T07:40:49,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:40:49,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779709114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779709121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779709123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779709180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779709190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779709230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779709241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779709241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-28T07:40:49,281 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:49,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-28T07:40:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T07:40:49,284 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:49,285 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:49,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T07:40:49,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779709390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779709400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T07:40:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:49,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6d263b57751c430489bf8696c66f2541 2024-11-28T07:40:49,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779709451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779709451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779709451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5d894a20ad604518a97af8150e90d94b is 50, key is test_row_0/B:col10/1732779647901/Put/seqid=0 2024-11-28T07:40:49,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741883_1059 (size=12301) 2024-11-28T07:40:49,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T07:40:49,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T07:40:49,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779709700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779709720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T07:40:49,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:49,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779709760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779709760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779709760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:49,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5d894a20ad604518a97af8150e90d94b 2024-11-28T07:40:49,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/4d60b8df69974e6185cedcff33121f3c is 50, key is test_row_0/C:col10/1732779647901/Put/seqid=0 2024-11-28T07:40:49,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741884_1060 (size=12301) 2024-11-28T07:40:49,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/4d60b8df69974e6185cedcff33121f3c 2024-11-28T07:40:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6d263b57751c430489bf8696c66f2541 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d263b57751c430489bf8696c66f2541 2024-11-28T07:40:49,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T07:40:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T07:40:49,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d263b57751c430489bf8696c66f2541, entries=150, sequenceid=270, filesize=12.0 K 2024-11-28T07:40:49,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5d894a20ad604518a97af8150e90d94b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5d894a20ad604518a97af8150e90d94b 2024-11-28T07:40:49,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5d894a20ad604518a97af8150e90d94b, entries=150, sequenceid=270, filesize=12.0 K 2024-11-28T07:40:49,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/4d60b8df69974e6185cedcff33121f3c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4d60b8df69974e6185cedcff33121f3c 2024-11-28T07:40:49,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4d60b8df69974e6185cedcff33121f3c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-28T07:40:49,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e36af693bfc3c757988aba93aff1bdda in 910ms, sequenceid=270, compaction requested=true 2024-11-28T07:40:49,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:49,937 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:49,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T07:40:49,938 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:49,939 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:49,939 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:49,939 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,939 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43b4da80b915459096484433d0ccd914, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/f9f728acb5f04d6386c6a89b4dd58432, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d263b57751c430489bf8696c66f2541] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.2 K 2024-11-28T07:40:49,940 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b4da80b915459096484433d0ccd914, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732779645988 2024-11-28T07:40:49,940 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:49,940 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:49,941 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:49,941 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4793dc2e24ef406daaf8140a79dce23e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/ca0da42f8fc44575a6d5bc5c1f597eed, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5d894a20ad604518a97af8150e90d94b] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.2 K 2024-11-28T07:40:49,941 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9f728acb5f04d6386c6a89b4dd58432, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732779646738 2024-11-28T07:40:49,942 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4793dc2e24ef406daaf8140a79dce23e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732779645988 2024-11-28T07:40:49,942 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d263b57751c430489bf8696c66f2541, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732779647901 2024-11-28T07:40:49,942 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0da42f8fc44575a6d5bc5c1f597eed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732779646738 2024-11-28T07:40:49,945 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d894a20ad604518a97af8150e90d94b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732779647901 2024-11-28T07:40:49,963 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:49,964 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/3415f5ed405d4552b84776ac3ad14364 is 50, key is test_row_0/B:col10/1732779647901/Put/seqid=0 2024-11-28T07:40:49,967 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:49,968 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6e8872a6c79d4932b37f48e4938036c2 is 50, key is test_row_0/A:col10/1732779647901/Put/seqid=0 2024-11-28T07:40:49,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741885_1061 (size=12847) 2024-11-28T07:40:49,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741886_1062 (size=12847) 2024-11-28T07:40:50,001 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/3415f5ed405d4552b84776ac3ad14364 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/3415f5ed405d4552b84776ac3ad14364 2024-11-28T07:40:50,001 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/6e8872a6c79d4932b37f48e4938036c2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6e8872a6c79d4932b37f48e4938036c2 2024-11-28T07:40:50,014 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into 6e8872a6c79d4932b37f48e4938036c2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:50,014 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 3415f5ed405d4552b84776ac3ad14364(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:50,014 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:50,014 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:50,014 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779649937; duration=0sec 2024-11-28T07:40:50,014 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779649937; duration=0sec 2024-11-28T07:40:50,014 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:50,014 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:50,014 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:50,014 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:50,014 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:50,017 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:50,017 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:50,017 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:50,017 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3c2c619660634a99bd2804e34b1c7538, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1b08af4b5f2c46c5a4853165b326bd3d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4d60b8df69974e6185cedcff33121f3c] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.2 K 2024-11-28T07:40:50,018 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c2c619660634a99bd2804e34b1c7538, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732779645988 2024-11-28T07:40:50,019 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b08af4b5f2c46c5a4853165b326bd3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732779646738 2024-11-28T07:40:50,019 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d60b8df69974e6185cedcff33121f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732779647901 2024-11-28T07:40:50,034 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:50,035 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/48515b1323404d1eba23c4beca47c37b is 50, key is test_row_0/C:col10/1732779647901/Put/seqid=0 2024-11-28T07:40:50,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741887_1063 (size=12847) 2024-11-28T07:40:50,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-28T07:40:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:50,057 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T07:40:50,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:50,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:50,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:50,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:50,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:50,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:50,060 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/48515b1323404d1eba23c4beca47c37b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/48515b1323404d1eba23c4beca47c37b 2024-11-28T07:40:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/928e7bd7efca4e8c90278eb92a803918 is 50, key is test_row_0/A:col10/1732779649121/Put/seqid=0 2024-11-28T07:40:50,072 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 48515b1323404d1eba23c4beca47c37b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:50,072 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:50,072 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779649937; duration=0sec 2024-11-28T07:40:50,072 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:50,072 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:50,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741888_1064 (size=12301) 2024-11-28T07:40:50,088 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/928e7bd7efca4e8c90278eb92a803918 2024-11-28T07:40:50,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/b10974046d834b09b2b1573df66b23d4 is 50, key is test_row_0/B:col10/1732779649121/Put/seqid=0 2024-11-28T07:40:50,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741889_1065 (size=12301) 2024-11-28T07:40:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:50,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:50,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779710239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779710241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779710270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779710270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779710271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779710350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779710351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T07:40:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/b10974046d834b09b2b1573df66b23d4 2024-11-28T07:40:50,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/30f1a456a75140e0acc922922288c694 is 50, key is test_row_0/C:col10/1732779649121/Put/seqid=0 2024-11-28T07:40:50,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741890_1066 (size=12301) 2024-11-28T07:40:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779710560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779710561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779710880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779710880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:50,934 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/30f1a456a75140e0acc922922288c694 2024-11-28T07:40:50,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/928e7bd7efca4e8c90278eb92a803918 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/928e7bd7efca4e8c90278eb92a803918 2024-11-28T07:40:50,949 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/928e7bd7efca4e8c90278eb92a803918, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T07:40:50,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/b10974046d834b09b2b1573df66b23d4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/b10974046d834b09b2b1573df66b23d4 2024-11-28T07:40:50,956 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/b10974046d834b09b2b1573df66b23d4, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T07:40:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/30f1a456a75140e0acc922922288c694 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/30f1a456a75140e0acc922922288c694 2024-11-28T07:40:50,966 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/30f1a456a75140e0acc922922288c694, entries=150, sequenceid=307, filesize=12.0 K 2024-11-28T07:40:50,967 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for e36af693bfc3c757988aba93aff1bdda in 911ms, sequenceid=307, compaction requested=false 2024-11-28T07:40:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:50,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-28T07:40:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-28T07:40:50,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T07:40:50,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6840 sec 2024-11-28T07:40:50,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6910 sec 2024-11-28T07:40:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:51,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:40:51,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:51,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:51,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:51,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:51,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:51,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:51,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/0242781c39e74bf29cf08a2ba75d2f33 is 50, key is test_row_0/A:col10/1732779650239/Put/seqid=0 2024-11-28T07:40:51,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741891_1067 (size=14741) 2024-11-28T07:40:51,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/0242781c39e74bf29cf08a2ba75d2f33 2024-11-28T07:40:51,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1bece14f350a46b8b8e1711c28c19bba is 50, key is test_row_0/B:col10/1732779650239/Put/seqid=0 2024-11-28T07:40:51,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741892_1068 (size=12301) 2024-11-28T07:40:51,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1bece14f350a46b8b8e1711c28c19bba 2024-11-28T07:40:51,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/3699d1ae1c9b417aac6783c9b01e1166 is 50, key is test_row_0/C:col10/1732779650239/Put/seqid=0 2024-11-28T07:40:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732779651313, max timestamp=1732779651331Num uniques 859; 2024-11-28T07:40:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779651317, max timestamp=1732779651331Num uniques 859; 2024-11-28T07:40:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779651317, max timestamp=1732779651331Num uniques 859; 2024-11-28T07:40:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-28T07:40:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:40:51,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:40:51,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741893_1069 (size=12301) 2024-11-28T07:40:51,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/3699d1ae1c9b417aac6783c9b01e1166 2024-11-28T07:40:51,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/0242781c39e74bf29cf08a2ba75d2f33 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/0242781c39e74bf29cf08a2ba75d2f33 2024-11-28T07:40:51,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/0242781c39e74bf29cf08a2ba75d2f33, entries=200, sequenceid=320, filesize=14.4 K 2024-11-28T07:40:51,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1bece14f350a46b8b8e1711c28c19bba as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1bece14f350a46b8b8e1711c28c19bba 2024-11-28T07:40:51,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779711375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1bece14f350a46b8b8e1711c28c19bba, entries=150, sequenceid=320, filesize=12.0 K 2024-11-28T07:40:51,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/3699d1ae1c9b417aac6783c9b01e1166 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3699d1ae1c9b417aac6783c9b01e1166 2024-11-28T07:40:51,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779711378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779711379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3699d1ae1c9b417aac6783c9b01e1166, entries=150, sequenceid=320, filesize=12.0 K 2024-11-28T07:40:51,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for e36af693bfc3c757988aba93aff1bdda in 109ms, sequenceid=320, compaction requested=true 2024-11-28T07:40:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:51,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:51,391 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:51,392 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:51,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:51,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:51,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:51,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:51,393 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:51,393 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:51,393 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,393 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6e8872a6c79d4932b37f48e4938036c2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/928e7bd7efca4e8c90278eb92a803918, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/0242781c39e74bf29cf08a2ba75d2f33] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=39.0 K 2024-11-28T07:40:51,393 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:51,393 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:51,393 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,393 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/3415f5ed405d4552b84776ac3ad14364, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/b10974046d834b09b2b1573df66b23d4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1bece14f350a46b8b8e1711c28c19bba] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.6 K 2024-11-28T07:40:51,394 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e8872a6c79d4932b37f48e4938036c2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732779647901 2024-11-28T07:40:51,394 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 3415f5ed405d4552b84776ac3ad14364, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732779647901 2024-11-28T07:40:51,394 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 928e7bd7efca4e8c90278eb92a803918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732779649104 2024-11-28T07:40:51,394 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting b10974046d834b09b2b1573df66b23d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732779649104 2024-11-28T07:40:51,394 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0242781c39e74bf29cf08a2ba75d2f33, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732779650232 2024-11-28T07:40:51,396 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bece14f350a46b8b8e1711c28c19bba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732779650232 2024-11-28T07:40:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:51,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T07:40:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:40:51,414 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:51,415 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c5cba1e208204e5e8978915f40e4615f is 50, key is test_row_0/B:col10/1732779650239/Put/seqid=0 2024-11-28T07:40:51,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5b759e93292540e5966ea08222ccb9de is 50, key is test_row_0/A:col10/1732779651363/Put/seqid=0 2024-11-28T07:40:51,418 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:51,418 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/050aa7e585804196a7465bdd20d2e354 is 50, key is test_row_0/A:col10/1732779650239/Put/seqid=0 2024-11-28T07:40:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-28T07:40:51,430 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:51,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-28T07:40:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779711426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779711428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,434 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:51,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,435 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:51,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T07:40:51,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741894_1070 (size=12949) 2024-11-28T07:40:51,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741896_1072 (size=12949) 2024-11-28T07:40:51,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741895_1071 (size=14741) 2024-11-28T07:40:51,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5b759e93292540e5966ea08222ccb9de 2024-11-28T07:40:51,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1b950c2786c34a7e86a9475f9680cdee is 50, key is test_row_0/B:col10/1732779651363/Put/seqid=0 2024-11-28T07:40:51,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741897_1073 (size=12301) 2024-11-28T07:40:51,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779711490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779711491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779711491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T07:40:51,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779711540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779711540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:51,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:51,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779711700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779711700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779711700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:51,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T07:40:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779711751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779711751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:51,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:51,868 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c5cba1e208204e5e8978915f40e4615f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c5cba1e208204e5e8978915f40e4615f 2024-11-28T07:40:51,871 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/050aa7e585804196a7465bdd20d2e354 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/050aa7e585804196a7465bdd20d2e354 2024-11-28T07:40:51,876 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into c5cba1e208204e5e8978915f40e4615f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:51,876 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:51,876 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779651391; duration=0sec 2024-11-28T07:40:51,876 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:51,876 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:51,877 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:51,878 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:51,878 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:51,878 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,878 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/48515b1323404d1eba23c4beca47c37b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/30f1a456a75140e0acc922922288c694, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3699d1ae1c9b417aac6783c9b01e1166] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.6 K 2024-11-28T07:40:51,879 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into 050aa7e585804196a7465bdd20d2e354(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:51,879 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:51,879 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779651391; duration=0sec 2024-11-28T07:40:51,879 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:51,879 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:51,879 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 48515b1323404d1eba23c4beca47c37b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732779647901 2024-11-28T07:40:51,879 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 30f1a456a75140e0acc922922288c694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732779649104 2024-11-28T07:40:51,880 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 3699d1ae1c9b417aac6783c9b01e1166, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732779650232 2024-11-28T07:40:51,889 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:51,890 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9734713a59ba4a62a5918decbd8b0ec6 is 50, key is test_row_0/C:col10/1732779650239/Put/seqid=0 2024-11-28T07:40:51,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1b950c2786c34a7e86a9475f9680cdee 2024-11-28T07:40:51,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741898_1074 (size=12949) 2024-11-28T07:40:51,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:51,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:51,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:51,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:51,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/6bb1fe4fa52443958b4eadd217c3638e is 50, key is test_row_0/C:col10/1732779651363/Put/seqid=0 2024-11-28T07:40:51,906 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9734713a59ba4a62a5918decbd8b0ec6 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9734713a59ba4a62a5918decbd8b0ec6 2024-11-28T07:40:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741899_1075 (size=12301) 2024-11-28T07:40:51,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/6bb1fe4fa52443958b4eadd217c3638e 2024-11-28T07:40:51,913 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 9734713a59ba4a62a5918decbd8b0ec6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:51,913 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:51,913 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779651392; duration=0sec 2024-11-28T07:40:51,913 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:51,914 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:51,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5b759e93292540e5966ea08222ccb9de as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5b759e93292540e5966ea08222ccb9de 2024-11-28T07:40:51,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5b759e93292540e5966ea08222ccb9de, entries=200, sequenceid=356, filesize=14.4 K 2024-11-28T07:40:51,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1b950c2786c34a7e86a9475f9680cdee as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1b950c2786c34a7e86a9475f9680cdee 2024-11-28T07:40:51,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1b950c2786c34a7e86a9475f9680cdee, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T07:40:51,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/6bb1fe4fa52443958b4eadd217c3638e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/6bb1fe4fa52443958b4eadd217c3638e 2024-11-28T07:40:51,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/6bb1fe4fa52443958b4eadd217c3638e, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T07:40:51,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for e36af693bfc3c757988aba93aff1bdda in 537ms, sequenceid=356, compaction requested=false 2024-11-28T07:40:51,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:52,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:52,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:40:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:52,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:52,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5010fd50e46240598de23790970387bf is 50, key is test_row_0/A:col10/1732779651417/Put/seqid=0 2024-11-28T07:40:52,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741900_1076 (size=14741) 2024-11-28T07:40:52,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T07:40:52,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779712060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779712060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779712065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779712066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779712066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779712181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779712181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779712181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:52,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:52,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779712390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779712390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779712390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5010fd50e46240598de23790970387bf 2024-11-28T07:40:52,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/414c8fba61184e50954343a782f75495 is 50, key is test_row_0/B:col10/1732779651417/Put/seqid=0 2024-11-28T07:40:52,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741901_1077 (size=12301) 2024-11-28T07:40:52,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:52,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T07:40:52,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779712571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779712580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779712700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779712700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779712700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:52,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/414c8fba61184e50954343a782f75495 2024-11-28T07:40:52,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/e158ea52a9be41409ccaa50388935bbe is 50, key is test_row_0/C:col10/1732779651417/Put/seqid=0 2024-11-28T07:40:52,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741902_1078 (size=12301) 2024-11-28T07:40:52,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:52,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:52,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:52,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:52,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:53,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:53,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:53,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:53,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:53,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779713210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779713210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779713210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/e158ea52a9be41409ccaa50388935bbe 2024-11-28T07:40:53,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5010fd50e46240598de23790970387bf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5010fd50e46240598de23790970387bf 2024-11-28T07:40:53,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5010fd50e46240598de23790970387bf, entries=200, sequenceid=372, filesize=14.4 K 2024-11-28T07:40:53,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/414c8fba61184e50954343a782f75495 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/414c8fba61184e50954343a782f75495 2024-11-28T07:40:53,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/414c8fba61184e50954343a782f75495, entries=150, sequenceid=372, filesize=12.0 K 2024-11-28T07:40:53,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/e158ea52a9be41409ccaa50388935bbe as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/e158ea52a9be41409ccaa50388935bbe 2024-11-28T07:40:53,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:53,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:53,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:53,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:53,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:53,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/e158ea52a9be41409ccaa50388935bbe, entries=150, sequenceid=372, filesize=12.0 K 2024-11-28T07:40:53,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e36af693bfc3c757988aba93aff1bdda in 1286ms, sequenceid=372, compaction requested=true 2024-11-28T07:40:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:53,300 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:53,300 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:53,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:53,302 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:53,302 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:53,302 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:53,302 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:53,302 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,302 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,302 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c5cba1e208204e5e8978915f40e4615f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1b950c2786c34a7e86a9475f9680cdee, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/414c8fba61184e50954343a782f75495] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.7 K 2024-11-28T07:40:53,302 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/050aa7e585804196a7465bdd20d2e354, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5b759e93292540e5966ea08222ccb9de, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5010fd50e46240598de23790970387bf] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=41.4 K 2024-11-28T07:40:53,302 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting c5cba1e208204e5e8978915f40e4615f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732779650232 2024-11-28T07:40:53,302 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050aa7e585804196a7465bdd20d2e354, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732779650232 2024-11-28T07:40:53,303 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b950c2786c34a7e86a9475f9680cdee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732779651363 2024-11-28T07:40:53,303 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b759e93292540e5966ea08222ccb9de, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732779651363 2024-11-28T07:40:53,303 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5010fd50e46240598de23790970387bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732779651417 2024-11-28T07:40:53,303 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 414c8fba61184e50954343a782f75495, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732779651417 2024-11-28T07:40:53,317 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#66 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:53,317 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:53,318 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/27c88feeca574c7ab56e77526c90906a is 50, key is test_row_0/A:col10/1732779651417/Put/seqid=0 2024-11-28T07:40:53,318 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/93930e6e7fa54b6195b868b0d20a377e is 50, key is test_row_0/B:col10/1732779651417/Put/seqid=0 2024-11-28T07:40:53,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741904_1080 (size=13051) 2024-11-28T07:40:53,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741903_1079 (size=13051) 2024-11-28T07:40:53,346 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/93930e6e7fa54b6195b868b0d20a377e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/93930e6e7fa54b6195b868b0d20a377e 2024-11-28T07:40:53,354 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 93930e6e7fa54b6195b868b0d20a377e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:53,355 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:53,355 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779653300; duration=0sec 2024-11-28T07:40:53,355 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:53,355 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:53,355 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:53,357 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:53,357 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:53,357 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,357 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9734713a59ba4a62a5918decbd8b0ec6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/6bb1fe4fa52443958b4eadd217c3638e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/e158ea52a9be41409ccaa50388935bbe] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.7 K 2024-11-28T07:40:53,358 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9734713a59ba4a62a5918decbd8b0ec6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732779650232 2024-11-28T07:40:53,358 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb1fe4fa52443958b4eadd217c3638e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732779651363 2024-11-28T07:40:53,359 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e158ea52a9be41409ccaa50388935bbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732779651417 2024-11-28T07:40:53,370 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:53,371 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/da26dbeaee694acab2126520a99ec8e2 is 50, key is test_row_0/C:col10/1732779651417/Put/seqid=0 2024-11-28T07:40:53,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741905_1081 (size=13051) 2024-11-28T07:40:53,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-28T07:40:53,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,444 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:40:53,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:53,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/43d0457e60dd4cf1bbde136d9a36e36f is 50, key is test_row_0/A:col10/1732779652039/Put/seqid=0 2024-11-28T07:40:53,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741906_1082 (size=12301) 2024-11-28T07:40:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T07:40:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:53,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:53,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779713607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779713609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779713723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779713730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,751 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/27c88feeca574c7ab56e77526c90906a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/27c88feeca574c7ab56e77526c90906a 2024-11-28T07:40:53,763 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into 27c88feeca574c7ab56e77526c90906a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:53,763 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:53,763 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779653300; duration=0sec 2024-11-28T07:40:53,763 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:53,763 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:53,793 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/da26dbeaee694acab2126520a99ec8e2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/da26dbeaee694acab2126520a99ec8e2 2024-11-28T07:40:53,800 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into da26dbeaee694acab2126520a99ec8e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:53,800 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:53,800 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779653300; duration=0sec 2024-11-28T07:40:53,800 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:53,800 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:53,858 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/43d0457e60dd4cf1bbde136d9a36e36f 2024-11-28T07:40:53,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5f30f3193fbf40f9adccd13044bba52c is 50, key is test_row_0/B:col10/1732779652039/Put/seqid=0 2024-11-28T07:40:53,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741907_1083 (size=12301) 2024-11-28T07:40:53,909 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5f30f3193fbf40f9adccd13044bba52c 2024-11-28T07:40:53,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9705481b8ddf476ba41e12d3d0baafdc is 50, key is test_row_0/C:col10/1732779652039/Put/seqid=0 2024-11-28T07:40:53,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741908_1084 (size=12301) 2024-11-28T07:40:53,937 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9705481b8ddf476ba41e12d3d0baafdc 2024-11-28T07:40:53,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779713940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779713943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/43d0457e60dd4cf1bbde136d9a36e36f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43d0457e60dd4cf1bbde136d9a36e36f 2024-11-28T07:40:53,968 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43d0457e60dd4cf1bbde136d9a36e36f, entries=150, sequenceid=392, filesize=12.0 K 2024-11-28T07:40:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5f30f3193fbf40f9adccd13044bba52c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5f30f3193fbf40f9adccd13044bba52c 2024-11-28T07:40:53,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5f30f3193fbf40f9adccd13044bba52c, entries=150, sequenceid=392, filesize=12.0 K 2024-11-28T07:40:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9705481b8ddf476ba41e12d3d0baafdc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9705481b8ddf476ba41e12d3d0baafdc 2024-11-28T07:40:53,991 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9705481b8ddf476ba41e12d3d0baafdc, entries=150, sequenceid=392, filesize=12.0 K 2024-11-28T07:40:53,992 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e36af693bfc3c757988aba93aff1bdda in 548ms, sequenceid=392, compaction requested=false 2024-11-28T07:40:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-28T07:40:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-28T07:40:53,998 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T07:40:53,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5590 sec 2024-11-28T07:40:54,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.5700 sec 2024-11-28T07:40:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:54,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:40:54,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:54,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:54,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:54,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:54,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:54,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:54,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/507cc332a1644416bce9c0b022a130b0 is 50, key is test_row_0/A:col10/1732779653594/Put/seqid=0 2024-11-28T07:40:54,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741909_1085 (size=14741) 2024-11-28T07:40:54,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/507cc332a1644416bce9c0b022a130b0 2024-11-28T07:40:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779714264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779714263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779714265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779714266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779714268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0a54e3b717a24eb8a588ffb97cb4cf17 is 50, key is test_row_0/B:col10/1732779653594/Put/seqid=0 2024-11-28T07:40:54,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741910_1086 (size=12301) 2024-11-28T07:40:54,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779714370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779714380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779714380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779714381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779714381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779714581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779714600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779714600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779714600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779714600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0a54e3b717a24eb8a588ffb97cb4cf17 2024-11-28T07:40:54,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/78cd1e50c41a42f0ac45be0e6b8fe965 is 50, key is test_row_0/C:col10/1732779653594/Put/seqid=0 2024-11-28T07:40:54,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741911_1087 (size=12301) 2024-11-28T07:40:54,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/78cd1e50c41a42f0ac45be0e6b8fe965 2024-11-28T07:40:54,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/507cc332a1644416bce9c0b022a130b0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/507cc332a1644416bce9c0b022a130b0 2024-11-28T07:40:54,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/507cc332a1644416bce9c0b022a130b0, entries=200, sequenceid=408, filesize=14.4 K 2024-11-28T07:40:54,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0a54e3b717a24eb8a588ffb97cb4cf17 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0a54e3b717a24eb8a588ffb97cb4cf17 2024-11-28T07:40:54,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0a54e3b717a24eb8a588ffb97cb4cf17, entries=150, sequenceid=408, filesize=12.0 K 2024-11-28T07:40:54,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/78cd1e50c41a42f0ac45be0e6b8fe965 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/78cd1e50c41a42f0ac45be0e6b8fe965 2024-11-28T07:40:54,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/78cd1e50c41a42f0ac45be0e6b8fe965, entries=150, sequenceid=408, filesize=12.0 K 2024-11-28T07:40:54,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e36af693bfc3c757988aba93aff1bdda in 547ms, sequenceid=408, compaction requested=true 2024-11-28T07:40:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:54,769 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:54,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:54,769 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:54,771 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:54,771 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:54,771 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:54,772 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/93930e6e7fa54b6195b868b0d20a377e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5f30f3193fbf40f9adccd13044bba52c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0a54e3b717a24eb8a588ffb97cb4cf17] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.8 K 2024-11-28T07:40:54,772 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:54,772 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:54,772 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:54,772 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/27c88feeca574c7ab56e77526c90906a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43d0457e60dd4cf1bbde136d9a36e36f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/507cc332a1644416bce9c0b022a130b0] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=39.2 K 2024-11-28T07:40:54,773 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 93930e6e7fa54b6195b868b0d20a377e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732779651417 2024-11-28T07:40:54,773 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27c88feeca574c7ab56e77526c90906a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732779651417 2024-11-28T07:40:54,774 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f30f3193fbf40f9adccd13044bba52c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732779652039 2024-11-28T07:40:54,774 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43d0457e60dd4cf1bbde136d9a36e36f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732779652039 2024-11-28T07:40:54,774 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a54e3b717a24eb8a588ffb97cb4cf17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732779653594 2024-11-28T07:40:54,774 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 507cc332a1644416bce9c0b022a130b0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732779653594 2024-11-28T07:40:54,803 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:54,804 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/4f97edbb46f9493db6bb11c7a2cde192 is 50, key is test_row_0/B:col10/1732779653594/Put/seqid=0 2024-11-28T07:40:54,809 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:54,809 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/618d48ec8a1449b79f9afcc29838a209 is 50, key is test_row_0/A:col10/1732779653594/Put/seqid=0 2024-11-28T07:40:54,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741912_1088 (size=13153) 2024-11-28T07:40:54,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741913_1089 (size=13153) 2024-11-28T07:40:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:54,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:40:54,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:54,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:54,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:54,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:54,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:54,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:54,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/051e7e351ee84dd6b4bd6ec1e3e9d994 is 50, key is test_row_0/A:col10/1732779654891/Put/seqid=0 2024-11-28T07:40:54,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741914_1090 (size=14741) 2024-11-28T07:40:54,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/051e7e351ee84dd6b4bd6ec1e3e9d994 2024-11-28T07:40:54,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/26e7a88b5b564c84a7528bad70dbb435 is 50, key is test_row_0/B:col10/1732779654891/Put/seqid=0 2024-11-28T07:40:54,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779714932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779714932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779714935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779714935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779714931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741915_1091 (size=12301) 2024-11-28T07:40:54,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/26e7a88b5b564c84a7528bad70dbb435 2024-11-28T07:40:54,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/c5ac50ec50164f3796e2c352894ee2b8 is 50, key is test_row_0/C:col10/1732779654891/Put/seqid=0 2024-11-28T07:40:54,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741916_1092 (size=12301) 2024-11-28T07:40:55,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779715060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779715060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779715060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779715060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779715060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,246 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/4f97edbb46f9493db6bb11c7a2cde192 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4f97edbb46f9493db6bb11c7a2cde192 2024-11-28T07:40:55,256 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/618d48ec8a1449b79f9afcc29838a209 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/618d48ec8a1449b79f9afcc29838a209 2024-11-28T07:40:55,265 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 4f97edbb46f9493db6bb11c7a2cde192(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:55,265 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:55,265 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779654769; duration=0sec 2024-11-28T07:40:55,265 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:55,266 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:55,266 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:55,267 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into 618d48ec8a1449b79f9afcc29838a209(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:55,267 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:55,267 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779654769; duration=0sec 2024-11-28T07:40:55,267 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:55,267 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:55,269 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:55,269 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:55,269 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:55,269 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/da26dbeaee694acab2126520a99ec8e2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9705481b8ddf476ba41e12d3d0baafdc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/78cd1e50c41a42f0ac45be0e6b8fe965] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.8 K 2024-11-28T07:40:55,270 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting da26dbeaee694acab2126520a99ec8e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732779651417 2024-11-28T07:40:55,271 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9705481b8ddf476ba41e12d3d0baafdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732779652039 2024-11-28T07:40:55,271 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 78cd1e50c41a42f0ac45be0e6b8fe965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732779653594 2024-11-28T07:40:55,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779715270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779715271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779715271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779715271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779715271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,284 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:55,284 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/ec7ae4fe85b9454cb428404b4bee2157 is 50, key is test_row_0/C:col10/1732779653594/Put/seqid=0 2024-11-28T07:40:55,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741917_1093 (size=13153) 2024-11-28T07:40:55,315 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/ec7ae4fe85b9454cb428404b4bee2157 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ec7ae4fe85b9454cb428404b4bee2157 2024-11-28T07:40:55,329 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into ec7ae4fe85b9454cb428404b4bee2157(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:55,329 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:55,329 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779654769; duration=0sec 2024-11-28T07:40:55,329 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:55,329 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:55,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/c5ac50ec50164f3796e2c352894ee2b8 2024-11-28T07:40:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/051e7e351ee84dd6b4bd6ec1e3e9d994 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/051e7e351ee84dd6b4bd6ec1e3e9d994 2024-11-28T07:40:55,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/051e7e351ee84dd6b4bd6ec1e3e9d994, entries=200, sequenceid=427, filesize=14.4 K 2024-11-28T07:40:55,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/26e7a88b5b564c84a7528bad70dbb435 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/26e7a88b5b564c84a7528bad70dbb435 2024-11-28T07:40:55,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/26e7a88b5b564c84a7528bad70dbb435, entries=150, sequenceid=427, filesize=12.0 K 2024-11-28T07:40:55,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/c5ac50ec50164f3796e2c352894ee2b8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c5ac50ec50164f3796e2c352894ee2b8 2024-11-28T07:40:55,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c5ac50ec50164f3796e2c352894ee2b8, entries=150, sequenceid=427, filesize=12.0 K 2024-11-28T07:40:55,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e36af693bfc3c757988aba93aff1bdda in 555ms, sequenceid=427, compaction requested=false 2024-11-28T07:40:55,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:55,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:40:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:55,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:55,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T07:40:55,601 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:55,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-28T07:40:55,606 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:55,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/dc618fc9bba64ddeab7b17bc640271c7 is 50, key is test_row_0/A:col10/1732779654931/Put/seqid=0 2024-11-28T07:40:55,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T07:40:55,612 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:55,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:55,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741918_1094 (size=12301) 2024-11-28T07:40:55,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779715635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779715639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/dc618fc9bba64ddeab7b17bc640271c7 2024-11-28T07:40:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779715641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779715643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779715644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/059519378a344d3284d2ac1156c61cc7 is 50, key is test_row_0/B:col10/1732779654931/Put/seqid=0 2024-11-28T07:40:55,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741919_1095 (size=12301) 2024-11-28T07:40:55,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/059519378a344d3284d2ac1156c61cc7 2024-11-28T07:40:55,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b29365e7b9804aa2924d9c2fc4cdc2ba is 50, key is test_row_0/C:col10/1732779654931/Put/seqid=0 2024-11-28T07:40:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T07:40:55,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779715750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741920_1096 (size=12301) 2024-11-28T07:40:55,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b29365e7b9804aa2924d9c2fc4cdc2ba 2024-11-28T07:40:55,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779715761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779715761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T07:40:55,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:55,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:55,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:55,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:55,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:55,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779715770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779715770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:55,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/dc618fc9bba64ddeab7b17bc640271c7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc618fc9bba64ddeab7b17bc640271c7 2024-11-28T07:40:55,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc618fc9bba64ddeab7b17bc640271c7, entries=150, sequenceid=444, filesize=12.0 K 2024-11-28T07:40:55,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/059519378a344d3284d2ac1156c61cc7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/059519378a344d3284d2ac1156c61cc7 2024-11-28T07:40:55,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/059519378a344d3284d2ac1156c61cc7, entries=150, sequenceid=444, filesize=12.0 K 2024-11-28T07:40:55,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b29365e7b9804aa2924d9c2fc4cdc2ba as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b29365e7b9804aa2924d9c2fc4cdc2ba 2024-11-28T07:40:55,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b29365e7b9804aa2924d9c2fc4cdc2ba, entries=150, sequenceid=444, filesize=12.0 K 2024-11-28T07:40:55,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for e36af693bfc3c757988aba93aff1bdda in 229ms, sequenceid=444, compaction requested=true 2024-11-28T07:40:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:55,815 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:55,815 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:55,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:55,817 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:55,817 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:55,817 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:55,817 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/618d48ec8a1449b79f9afcc29838a209, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/051e7e351ee84dd6b4bd6ec1e3e9d994, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc618fc9bba64ddeab7b17bc640271c7] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=39.3 K 2024-11-28T07:40:55,818 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:55,818 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:55,818 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:55,818 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 618d48ec8a1449b79f9afcc29838a209, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732779653594 2024-11-28T07:40:55,818 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4f97edbb46f9493db6bb11c7a2cde192, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/26e7a88b5b564c84a7528bad70dbb435, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/059519378a344d3284d2ac1156c61cc7] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.9 K 2024-11-28T07:40:55,818 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 051e7e351ee84dd6b4bd6ec1e3e9d994, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732779654250 2024-11-28T07:40:55,818 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f97edbb46f9493db6bb11c7a2cde192, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732779653594 2024-11-28T07:40:55,819 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e7a88b5b564c84a7528bad70dbb435, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732779654250 2024-11-28T07:40:55,819 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc618fc9bba64ddeab7b17bc640271c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732779654929 2024-11-28T07:40:55,820 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 059519378a344d3284d2ac1156c61cc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732779654929 2024-11-28T07:40:55,847 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:55,847 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/ad3eddf319fe44e891222f9e65deb125 is 50, key is test_row_0/A:col10/1732779654931/Put/seqid=0 2024-11-28T07:40:55,852 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:55,854 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0239b731628f40719c929e58ca83d3b0 is 50, key is test_row_0/B:col10/1732779654931/Put/seqid=0 2024-11-28T07:40:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741922_1098 (size=13255) 2024-11-28T07:40:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T07:40:55,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-28T07:40:55,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:55,965 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:40:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:55,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:55,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:55,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:55,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741921_1097 (size=13255) 2024-11-28T07:40:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:55,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/4d89c09e3ffd4665b4576a273495f1ff is 50, key is test_row_0/A:col10/1732779655632/Put/seqid=0 2024-11-28T07:40:55,983 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/ad3eddf319fe44e891222f9e65deb125 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/ad3eddf319fe44e891222f9e65deb125 2024-11-28T07:40:55,991 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into ad3eddf319fe44e891222f9e65deb125(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:55,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:55,991 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779655815; duration=0sec 2024-11-28T07:40:55,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:55,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:55,991 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:55,993 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:55,993 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:55,993 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:55,994 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ec7ae4fe85b9454cb428404b4bee2157, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c5ac50ec50164f3796e2c352894ee2b8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b29365e7b9804aa2924d9c2fc4cdc2ba] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=36.9 K 2024-11-28T07:40:55,994 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7ae4fe85b9454cb428404b4bee2157, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1732779653594 2024-11-28T07:40:55,995 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ac50ec50164f3796e2c352894ee2b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732779654250 2024-11-28T07:40:55,995 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting b29365e7b9804aa2924d9c2fc4cdc2ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732779654929 2024-11-28T07:40:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779715993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779715994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779715995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779716005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779716007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,015 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:56,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741923_1099 (size=12301) 2024-11-28T07:40:56,016 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/43033f17de7049c3ba332b622d42b2cc is 50, key is test_row_0/C:col10/1732779654931/Put/seqid=0 2024-11-28T07:40:56,020 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/4d89c09e3ffd4665b4576a273495f1ff 2024-11-28T07:40:56,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1621421365484d418d5047c796a907bc is 50, key is test_row_0/B:col10/1732779655632/Put/seqid=0 2024-11-28T07:40:56,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741924_1100 (size=13255) 2024-11-28T07:40:56,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741925_1101 (size=12301) 2024-11-28T07:40:56,073 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1621421365484d418d5047c796a907bc 2024-11-28T07:40:56,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/5e0a7409cee344abab5a26718403093a is 50, key is test_row_0/C:col10/1732779655632/Put/seqid=0 2024-11-28T07:40:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779716120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779716120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779716120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779716120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779716121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741926_1102 (size=12301) 2024-11-28T07:40:56,159 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/5e0a7409cee344abab5a26718403093a 2024-11-28T07:40:56,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/4d89c09e3ffd4665b4576a273495f1ff as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d89c09e3ffd4665b4576a273495f1ff 2024-11-28T07:40:56,184 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d89c09e3ffd4665b4576a273495f1ff, entries=150, sequenceid=463, filesize=12.0 K 2024-11-28T07:40:56,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1621421365484d418d5047c796a907bc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1621421365484d418d5047c796a907bc 2024-11-28T07:40:56,213 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1621421365484d418d5047c796a907bc, entries=150, sequenceid=463, filesize=12.0 K 2024-11-28T07:40:56,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/5e0a7409cee344abab5a26718403093a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/5e0a7409cee344abab5a26718403093a 2024-11-28T07:40:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T07:40:56,232 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/5e0a7409cee344abab5a26718403093a, entries=150, sequenceid=463, filesize=12.0 K 2024-11-28T07:40:56,234 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e36af693bfc3c757988aba93aff1bdda in 312ms, sequenceid=463, compaction requested=false 2024-11-28T07:40:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:56,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-28T07:40:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-28T07:40:56,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-28T07:40:56,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 624 msec 2024-11-28T07:40:56,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 639 msec 2024-11-28T07:40:56,318 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/0239b731628f40719c929e58ca83d3b0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0239b731628f40719c929e58ca83d3b0 2024-11-28T07:40:56,328 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 0239b731628f40719c929e58ca83d3b0(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:56,328 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:56,328 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779655815; duration=0sec 2024-11-28T07:40:56,328 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:56,328 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:56,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:40:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:56,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/70a9e97e313f4a71bb2b9b231f6c7bfe is 50, key is test_row_0/A:col10/1732779656005/Put/seqid=0 2024-11-28T07:40:56,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741927_1103 (size=12301) 2024-11-28T07:40:56,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779716379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779716394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,450 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,460 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/43033f17de7049c3ba332b622d42b2cc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/43033f17de7049c3ba332b622d42b2cc 2024-11-28T07:40:56,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779716449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779716449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779716444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,469 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 43033f17de7049c3ba332b622d42b2cc(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:56,469 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:56,469 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779655815; duration=0sec 2024-11-28T07:40:56,469 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:56,469 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:56,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779716550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779716560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779716571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779716571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779716580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-28T07:40:56,741 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:56,742 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-28T07:40:56,750 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:56,751 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:56,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T07:40:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779716760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779716770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/70a9e97e313f4a71bb2b9b231f6c7bfe 2024-11-28T07:40:56,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779716780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779716790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779716791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:56,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1ff6b09535ef48ceab623a360e63fb25 is 50, key is test_row_0/B:col10/1732779656005/Put/seqid=0 2024-11-28T07:40:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T07:40:56,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741928_1104 (size=12301) 2024-11-28T07:40:56,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T07:40:56,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:56,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:56,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:56,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:56,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:56,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:57,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T07:40:57,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:57,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T07:40:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779717070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779717081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,087 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779717100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779717111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779717110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T07:40:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:57,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1ff6b09535ef48ceab623a360e63fb25 2024-11-28T07:40:57,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/ef61014866f943438397564ddf22f0f0 is 50, key is test_row_0/C:col10/1732779656005/Put/seqid=0 2024-11-28T07:40:57,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741929_1105 (size=12301) 2024-11-28T07:40:57,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/ef61014866f943438397564ddf22f0f0 2024-11-28T07:40:57,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/70a9e97e313f4a71bb2b9b231f6c7bfe as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/70a9e97e313f4a71bb2b9b231f6c7bfe 2024-11-28T07:40:57,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/70a9e97e313f4a71bb2b9b231f6c7bfe, entries=150, sequenceid=480, filesize=12.0 K 2024-11-28T07:40:57,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/1ff6b09535ef48ceab623a360e63fb25 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1ff6b09535ef48ceab623a360e63fb25 2024-11-28T07:40:57,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1ff6b09535ef48ceab623a360e63fb25, entries=150, sequenceid=480, filesize=12.0 K 2024-11-28T07:40:57,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/ef61014866f943438397564ddf22f0f0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ef61014866f943438397564ddf22f0f0 2024-11-28T07:40:57,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ef61014866f943438397564ddf22f0f0, entries=150, sequenceid=480, filesize=12.0 K 2024-11-28T07:40:57,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e36af693bfc3c757988aba93aff1bdda in 1020ms, sequenceid=480, compaction requested=true 2024-11-28T07:40:57,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:57,365 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:57,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:57,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:57,365 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:57,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:57,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:57,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:57,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:57,367 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:57,367 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:57,367 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,367 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0239b731628f40719c929e58ca83d3b0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1621421365484d418d5047c796a907bc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1ff6b09535ef48ceab623a360e63fb25] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.0 K 2024-11-28T07:40:57,368 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0239b731628f40719c929e58ca83d3b0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732779654929 2024-11-28T07:40:57,370 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:57,370 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:57,371 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,371 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/ad3eddf319fe44e891222f9e65deb125, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d89c09e3ffd4665b4576a273495f1ff, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/70a9e97e313f4a71bb2b9b231f6c7bfe] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.0 K 2024-11-28T07:40:57,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-28T07:40:57,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,372 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:40:57,372 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad3eddf319fe44e891222f9e65deb125, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732779654929 2024-11-28T07:40:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:57,373 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d89c09e3ffd4665b4576a273495f1ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732779655632 2024-11-28T07:40:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:57,373 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1621421365484d418d5047c796a907bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732779655632 2024-11-28T07:40:57,373 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70a9e97e313f4a71bb2b9b231f6c7bfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732779655995 2024-11-28T07:40:57,374 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ff6b09535ef48ceab623a360e63fb25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732779655995 2024-11-28T07:40:57,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/b1c1e7432e9744fe876c691f89cf36d9 is 50, key is test_row_0/A:col10/1732779656394/Put/seqid=0 2024-11-28T07:40:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T07:40:57,391 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:57,392 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/f680785383a0456f9d39439c4bfc74d7 is 50, key is test_row_0/B:col10/1732779656005/Put/seqid=0 2024-11-28T07:40:57,398 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:57,399 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/16ab867bb84a4d0cb95aadeaaec9f0e7 is 50, key is test_row_0/A:col10/1732779656005/Put/seqid=0 2024-11-28T07:40:57,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741930_1106 (size=12301) 2024-11-28T07:40:57,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741931_1107 (size=13357) 2024-11-28T07:40:57,425 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/f680785383a0456f9d39439c4bfc74d7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/f680785383a0456f9d39439c4bfc74d7 2024-11-28T07:40:57,434 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into f680785383a0456f9d39439c4bfc74d7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:57,434 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:57,434 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779657365; duration=0sec 2024-11-28T07:40:57,434 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:57,435 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:57,435 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:57,437 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:57,437 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:57,437 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,437 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/43033f17de7049c3ba332b622d42b2cc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/5e0a7409cee344abab5a26718403093a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ef61014866f943438397564ddf22f0f0] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.0 K 2024-11-28T07:40:57,438 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 43033f17de7049c3ba332b622d42b2cc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1732779654929 2024-11-28T07:40:57,438 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0a7409cee344abab5a26718403093a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732779655632 2024-11-28T07:40:57,438 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ef61014866f943438397564ddf22f0f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732779655995 2024-11-28T07:40:57,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741932_1108 (size=13357) 2024-11-28T07:40:57,466 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:57,466 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/16ab867bb84a4d0cb95aadeaaec9f0e7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/16ab867bb84a4d0cb95aadeaaec9f0e7 2024-11-28T07:40:57,467 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9ccf230097ad4d54869cf4bdbe73cf8f is 50, key is test_row_0/C:col10/1732779656005/Put/seqid=0 2024-11-28T07:40:57,478 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into 16ab867bb84a4d0cb95aadeaaec9f0e7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:57,478 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:57,478 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779657365; duration=0sec 2024-11-28T07:40:57,478 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:57,478 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:57,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741933_1109 (size=13357) 2024-11-28T07:40:57,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:57,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:57,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779717621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779717630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779717623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779717630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779717630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779717740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:57,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779717741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:57,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:57,808 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/b1c1e7432e9744fe876c691f89cf36d9 2024-11-28T07:40:57,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/8ace933579564b2e9fadce2254729e8e is 50, key is test_row_0/B:col10/1732779656394/Put/seqid=0 2024-11-28T07:40:57,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741934_1110 (size=12301) 2024-11-28T07:40:57,854 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/8ace933579564b2e9fadce2254729e8e 2024-11-28T07:40:57,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1da47a4c414449cb8b03a3b85fcc729c is 50, key is test_row_0/C:col10/1732779656394/Put/seqid=0 2024-11-28T07:40:57,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741935_1111 (size=12301) 2024-11-28T07:40:57,888 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1da47a4c414449cb8b03a3b85fcc729c 2024-11-28T07:40:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T07:40:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/b1c1e7432e9744fe876c691f89cf36d9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/b1c1e7432e9744fe876c691f89cf36d9 2024-11-28T07:40:57,911 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/b1c1e7432e9744fe876c691f89cf36d9, entries=150, sequenceid=499, filesize=12.0 K 2024-11-28T07:40:57,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/8ace933579564b2e9fadce2254729e8e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/8ace933579564b2e9fadce2254729e8e 2024-11-28T07:40:57,920 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/8ace933579564b2e9fadce2254729e8e, entries=150, sequenceid=499, filesize=12.0 K 2024-11-28T07:40:57,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/1da47a4c414449cb8b03a3b85fcc729c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1da47a4c414449cb8b03a3b85fcc729c 2024-11-28T07:40:57,927 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1da47a4c414449cb8b03a3b85fcc729c, entries=150, sequenceid=499, filesize=12.0 K 2024-11-28T07:40:57,928 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e36af693bfc3c757988aba93aff1bdda in 556ms, sequenceid=499, compaction requested=false 2024-11-28T07:40:57,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:57,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:57,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-28T07:40:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-28T07:40:57,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-28T07:40:57,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1800 sec 2024-11-28T07:40:57,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1940 sec 2024-11-28T07:40:57,947 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/9ccf230097ad4d54869cf4bdbe73cf8f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9ccf230097ad4d54869cf4bdbe73cf8f 2024-11-28T07:40:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:57,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:40:57,960 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 9ccf230097ad4d54869cf4bdbe73cf8f(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:57,960 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:57,960 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779657366; duration=0sec 2024-11-28T07:40:57,960 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:57,960 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:57,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:57,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:57,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:57,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:57,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:57,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:57,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/4d068e6804ef4ec5aa9da90c10938b3e is 50, key is test_row_0/A:col10/1732779657609/Put/seqid=0 2024-11-28T07:40:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779718006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779718015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741936_1112 (size=14741) 2024-11-28T07:40:58,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/4d068e6804ef4ec5aa9da90c10938b3e 2024-11-28T07:40:58,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c4e4eb9ebb3645cea65da40d9e3f8535 is 50, key is test_row_0/B:col10/1732779657609/Put/seqid=0 2024-11-28T07:40:58,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741937_1113 (size=12301) 2024-11-28T07:40:58,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c4e4eb9ebb3645cea65da40d9e3f8535 2024-11-28T07:40:58,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/0ca973a83d494dd48400ef5b80343476 is 50, key is test_row_0/C:col10/1732779657609/Put/seqid=0 2024-11-28T07:40:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779718120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779718130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741938_1114 (size=12301) 2024-11-28T07:40:58,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=518 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/0ca973a83d494dd48400ef5b80343476 2024-11-28T07:40:58,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/4d068e6804ef4ec5aa9da90c10938b3e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d068e6804ef4ec5aa9da90c10938b3e 2024-11-28T07:40:58,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d068e6804ef4ec5aa9da90c10938b3e, entries=200, sequenceid=518, filesize=14.4 K 2024-11-28T07:40:58,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/c4e4eb9ebb3645cea65da40d9e3f8535 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c4e4eb9ebb3645cea65da40d9e3f8535 2024-11-28T07:40:58,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c4e4eb9ebb3645cea65da40d9e3f8535, entries=150, sequenceid=518, filesize=12.0 K 2024-11-28T07:40:58,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/0ca973a83d494dd48400ef5b80343476 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/0ca973a83d494dd48400ef5b80343476 2024-11-28T07:40:58,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/0ca973a83d494dd48400ef5b80343476, entries=150, sequenceid=518, filesize=12.0 K 2024-11-28T07:40:58,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for e36af693bfc3c757988aba93aff1bdda in 233ms, sequenceid=518, compaction requested=true 2024-11-28T07:40:58,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,187 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:58,188 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:58,188 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:58,188 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:58,188 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/16ab867bb84a4d0cb95aadeaaec9f0e7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/b1c1e7432e9744fe876c691f89cf36d9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d068e6804ef4ec5aa9da90c10938b3e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=39.5 K 2024-11-28T07:40:58,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:40:58,189 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16ab867bb84a4d0cb95aadeaaec9f0e7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732779655995 2024-11-28T07:40:58,189 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c1e7432e9744fe876c691f89cf36d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1732779656378 2024-11-28T07:40:58,190 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d068e6804ef4ec5aa9da90c10938b3e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732779657609 2024-11-28T07:40:58,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:58,191 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:58,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:58,193 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:58,193 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:58,193 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:58,193 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/f680785383a0456f9d39439c4bfc74d7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/8ace933579564b2e9fadce2254729e8e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c4e4eb9ebb3645cea65da40d9e3f8535] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.1 K 2024-11-28T07:40:58,194 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f680785383a0456f9d39439c4bfc74d7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732779655995 2024-11-28T07:40:58,194 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ace933579564b2e9fadce2254729e8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1732779656378 2024-11-28T07:40:58,195 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e4eb9ebb3645cea65da40d9e3f8535, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732779657609 2024-11-28T07:40:58,211 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:58,211 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/aec703b1d87b4338812d6c4b0a254a3f is 50, key is test_row_0/A:col10/1732779657609/Put/seqid=0 2024-11-28T07:40:58,212 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:58,212 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5c50655f44514c3ba6702c1997d4421e is 50, key is test_row_0/B:col10/1732779657609/Put/seqid=0 2024-11-28T07:40:58,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741940_1116 (size=13459) 2024-11-28T07:40:58,252 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/5c50655f44514c3ba6702c1997d4421e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5c50655f44514c3ba6702c1997d4421e 2024-11-28T07:40:58,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741939_1115 (size=13459) 2024-11-28T07:40:58,261 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/aec703b1d87b4338812d6c4b0a254a3f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aec703b1d87b4338812d6c4b0a254a3f 2024-11-28T07:40:58,262 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 5c50655f44514c3ba6702c1997d4421e(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:58,262 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,262 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779658191; duration=0sec 2024-11-28T07:40:58,262 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:58,262 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:58,262 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:58,264 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:58,264 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:58,264 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:58,265 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9ccf230097ad4d54869cf4bdbe73cf8f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1da47a4c414449cb8b03a3b85fcc729c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/0ca973a83d494dd48400ef5b80343476] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.1 K 2024-11-28T07:40:58,265 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ccf230097ad4d54869cf4bdbe73cf8f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732779655995 2024-11-28T07:40:58,267 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1da47a4c414449cb8b03a3b85fcc729c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1732779656378 2024-11-28T07:40:58,268 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ca973a83d494dd48400ef5b80343476, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732779657609 2024-11-28T07:40:58,269 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into aec703b1d87b4338812d6c4b0a254a3f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:58,269 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,269 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779658187; duration=0sec 2024-11-28T07:40:58,269 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:58,270 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:58,287 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:58,288 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/3af0497c7e6c45d387c9b416fa2e67ca is 50, key is test_row_0/C:col10/1732779657609/Put/seqid=0 2024-11-28T07:40:58,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741941_1117 (size=13459) 2024-11-28T07:40:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:58,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:40:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/7c23c465a2174e5e87efb6e9dbc4ea25 is 50, key is test_row_0/A:col10/1732779658012/Put/seqid=0 2024-11-28T07:40:58,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741942_1118 (size=14741) 2024-11-28T07:40:58,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/7c23c465a2174e5e87efb6e9dbc4ea25 2024-11-28T07:40:58,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2ff0638a5be04345bd7831931449537d is 50, key is test_row_0/B:col10/1732779658012/Put/seqid=0 2024-11-28T07:40:58,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779718375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779718382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741943_1119 (size=12301) 2024-11-28T07:40:58,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2ff0638a5be04345bd7831931449537d 2024-11-28T07:40:58,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/2822a28913b1484d898ae6726eebfebd is 50, key is test_row_0/C:col10/1732779658012/Put/seqid=0 2024-11-28T07:40:58,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741944_1120 (size=12301) 2024-11-28T07:40:58,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/2822a28913b1484d898ae6726eebfebd 2024-11-28T07:40:58,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/7c23c465a2174e5e87efb6e9dbc4ea25 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/7c23c465a2174e5e87efb6e9dbc4ea25 2024-11-28T07:40:58,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/7c23c465a2174e5e87efb6e9dbc4ea25, entries=200, sequenceid=537, filesize=14.4 K 2024-11-28T07:40:58,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/2ff0638a5be04345bd7831931449537d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2ff0638a5be04345bd7831931449537d 2024-11-28T07:40:58,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2ff0638a5be04345bd7831931449537d, entries=150, sequenceid=537, filesize=12.0 K 2024-11-28T07:40:58,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/2822a28913b1484d898ae6726eebfebd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2822a28913b1484d898ae6726eebfebd 2024-11-28T07:40:58,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2822a28913b1484d898ae6726eebfebd, entries=150, sequenceid=537, filesize=12.0 K 2024-11-28T07:40:58,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e36af693bfc3c757988aba93aff1bdda in 160ms, sequenceid=537, compaction requested=false 2024-11-28T07:40:58,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:58,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:40:58,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:58,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:58,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:58,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5e023d7a81314ed5bd9a9ec7d35f1e35 is 50, key is test_row_0/A:col10/1732779658360/Put/seqid=0 2024-11-28T07:40:58,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741945_1121 (size=14741) 2024-11-28T07:40:58,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5e023d7a81314ed5bd9a9ec7d35f1e35 2024-11-28T07:40:58,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779718562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/03fb16e8069847bf8bea7e180d25ac97 is 50, key is test_row_0/B:col10/1732779658360/Put/seqid=0 2024-11-28T07:40:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779718566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741946_1122 (size=12301) 2024-11-28T07:40:58,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/03fb16e8069847bf8bea7e180d25ac97 2024-11-28T07:40:58,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/418f6862921b4162a4371d3b3ade5b87 is 50, key is test_row_0/C:col10/1732779658360/Put/seqid=0 2024-11-28T07:40:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741947_1123 (size=12301) 2024-11-28T07:40:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56032 deadline: 1732779718640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,646 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in e36af693bfc3c757988aba93aff1bdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:40:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/418f6862921b4162a4371d3b3ade5b87 2024-11-28T07:40:58,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56024 deadline: 1732779718650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56036 deadline: 1732779718650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,653 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in e36af693bfc3c757988aba93aff1bdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,654 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in e36af693bfc3c757988aba93aff1bdda of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:40:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5e023d7a81314ed5bd9a9ec7d35f1e35 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5e023d7a81314ed5bd9a9ec7d35f1e35 2024-11-28T07:40:58,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5e023d7a81314ed5bd9a9ec7d35f1e35, entries=200, sequenceid=554, filesize=14.4 K 2024-11-28T07:40:58,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/03fb16e8069847bf8bea7e180d25ac97 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/03fb16e8069847bf8bea7e180d25ac97 2024-11-28T07:40:58,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/03fb16e8069847bf8bea7e180d25ac97, entries=150, sequenceid=554, filesize=12.0 K 2024-11-28T07:40:58,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/418f6862921b4162a4371d3b3ade5b87 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/418f6862921b4162a4371d3b3ade5b87 2024-11-28T07:40:58,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/418f6862921b4162a4371d3b3ade5b87, entries=150, sequenceid=554, filesize=12.0 K 2024-11-28T07:40:58,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e36af693bfc3c757988aba93aff1bdda in 177ms, sequenceid=554, compaction requested=true 2024-11-28T07:40:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:40:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:58,684 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:40:58,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T07:40:58,685 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:58,685 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/A is initiating minor compaction (all files) 2024-11-28T07:40:58,685 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/A in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:58,685 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aec703b1d87b4338812d6c4b0a254a3f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/7c23c465a2174e5e87efb6e9dbc4ea25, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5e023d7a81314ed5bd9a9ec7d35f1e35] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=41.9 K 2024-11-28T07:40:58,686 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec703b1d87b4338812d6c4b0a254a3f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732779657609 2024-11-28T07:40:58,687 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c23c465a2174e5e87efb6e9dbc4ea25, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732779657993 2024-11-28T07:40:58,687 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e023d7a81314ed5bd9a9ec7d35f1e35, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1732779658360 2024-11-28T07:40:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:58,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:40:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:58,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:58,700 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#A#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:58,700 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/cab75ca7067e4e9aa4d801f8a48fa4d0 is 50, key is test_row_0/A:col10/1732779658360/Put/seqid=0 2024-11-28T07:40:58,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/da2074e75f36434f860aaebfbf577357 is 50, key is test_row_0/A:col10/1732779658685/Put/seqid=0 2024-11-28T07:40:58,715 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/3af0497c7e6c45d387c9b416fa2e67ca as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3af0497c7e6c45d387c9b416fa2e67ca 2024-11-28T07:40:58,725 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into 3af0497c7e6c45d387c9b416fa2e67ca(size=13.1 K), total size for store is 37.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:58,725 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,725 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779658193; duration=0sec 2024-11-28T07:40:58,725 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T07:40:58,725 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:58,725 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:58,725 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:58,727 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:58,727 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/B is initiating minor compaction (all files) 2024-11-28T07:40:58,727 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/B in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:58,727 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5c50655f44514c3ba6702c1997d4421e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2ff0638a5be04345bd7831931449537d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/03fb16e8069847bf8bea7e180d25ac97] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.2 K 2024-11-28T07:40:58,727 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c50655f44514c3ba6702c1997d4421e, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732779657609 2024-11-28T07:40:58,728 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff0638a5be04345bd7831931449537d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732779657993 2024-11-28T07:40:58,728 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 03fb16e8069847bf8bea7e180d25ac97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1732779658360 2024-11-28T07:40:58,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779718742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779718751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741949_1125 (size=17181) 2024-11-28T07:40:58,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741948_1124 (size=13561) 2024-11-28T07:40:58,794 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#B#compaction#113 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:58,795 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/62269353f8ad47c08f94a9c9e96ae956 is 50, key is test_row_0/B:col10/1732779658360/Put/seqid=0 2024-11-28T07:40:58,802 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/cab75ca7067e4e9aa4d801f8a48fa4d0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/cab75ca7067e4e9aa4d801f8a48fa4d0 2024-11-28T07:40:58,810 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/A of e36af693bfc3c757988aba93aff1bdda into cab75ca7067e4e9aa4d801f8a48fa4d0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:58,810 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,810 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/A, priority=13, startTime=1732779658684; duration=0sec 2024-11-28T07:40:58,811 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:40:58,811 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:40:58,811 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:40:58,814 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:40:58,814 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): e36af693bfc3c757988aba93aff1bdda/C is initiating minor compaction (all files) 2024-11-28T07:40:58,814 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36af693bfc3c757988aba93aff1bdda/C in TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:58,814 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3af0497c7e6c45d387c9b416fa2e67ca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2822a28913b1484d898ae6726eebfebd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/418f6862921b4162a4371d3b3ade5b87] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp, totalSize=37.2 K 2024-11-28T07:40:58,815 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3af0497c7e6c45d387c9b416fa2e67ca, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=518, earliestPutTs=1732779657609 2024-11-28T07:40:58,815 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2822a28913b1484d898ae6726eebfebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732779657993 2024-11-28T07:40:58,816 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 418f6862921b4162a4371d3b3ade5b87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=554, earliestPutTs=1732779658360 2024-11-28T07:40:58,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741950_1126 (size=13561) 2024-11-28T07:40:58,853 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36af693bfc3c757988aba93aff1bdda#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:40:58,853 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/c405b6591dfb4bdcb49e58f8258e5f51 is 50, key is test_row_0/C:col10/1732779658360/Put/seqid=0 2024-11-28T07:40:58,853 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/62269353f8ad47c08f94a9c9e96ae956 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/62269353f8ad47c08f94a9c9e96ae956 2024-11-28T07:40:58,864 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/B of e36af693bfc3c757988aba93aff1bdda into 62269353f8ad47c08f94a9c9e96ae956(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:58,864 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,864 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/B, priority=13, startTime=1732779658684; duration=0sec 2024-11-28T07:40:58,865 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:58,865 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:40:58,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779718860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779718870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:58,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741951_1127 (size=13561) 2024-11-28T07:40:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-28T07:40:58,900 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:40:58,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:40:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-28T07:40:58,903 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:40:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T07:40:58,905 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:40:58,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:40:58,936 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/c405b6591dfb4bdcb49e58f8258e5f51 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c405b6591dfb4bdcb49e58f8258e5f51 2024-11-28T07:40:58,946 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e36af693bfc3c757988aba93aff1bdda/C of e36af693bfc3c757988aba93aff1bdda into c405b6591dfb4bdcb49e58f8258e5f51(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:40:58,947 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:58,947 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., storeName=e36af693bfc3c757988aba93aff1bdda/C, priority=13, startTime=1732779658684; duration=0sec 2024-11-28T07:40:58,947 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:40:58,947 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:40:59,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T07:40:59,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T07:40:59,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:59,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779719080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:59,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779719093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:59,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/da2074e75f36434f860aaebfbf577357 2024-11-28T07:40:59,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/77167bb3dd844d18994bf12be1d38697 is 50, key is test_row_0/B:col10/1732779658685/Put/seqid=0 2024-11-28T07:40:59,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741952_1128 (size=12301) 2024-11-28T07:40:59,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/77167bb3dd844d18994bf12be1d38697 2024-11-28T07:40:59,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T07:40:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:59,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b0537145519d4d00a6875150f7ec2998 is 50, key is test_row_0/C:col10/1732779658685/Put/seqid=0 2024-11-28T07:40:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T07:40:59,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741953_1129 (size=12301) 2024-11-28T07:40:59,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T07:40:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779719390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:59,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779719413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:59,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T07:40:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T07:40:59,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b0537145519d4d00a6875150f7ec2998 2024-11-28T07:40:59,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T07:40:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:40:59,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/da2074e75f36434f860aaebfbf577357 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/da2074e75f36434f860aaebfbf577357 2024-11-28T07:40:59,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/da2074e75f36434f860aaebfbf577357, entries=250, sequenceid=571, filesize=16.8 K 2024-11-28T07:40:59,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/77167bb3dd844d18994bf12be1d38697 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/77167bb3dd844d18994bf12be1d38697 2024-11-28T07:40:59,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/77167bb3dd844d18994bf12be1d38697, entries=150, sequenceid=571, filesize=12.0 K 2024-11-28T07:40:59,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/b0537145519d4d00a6875150f7ec2998 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b0537145519d4d00a6875150f7ec2998 2024-11-28T07:40:59,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b0537145519d4d00a6875150f7ec2998, entries=150, sequenceid=571, filesize=12.0 K 2024-11-28T07:40:59,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e36af693bfc3c757988aba93aff1bdda in 1015ms, sequenceid=571, compaction requested=false 2024-11-28T07:40:59,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:40:59,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-28T07:40:59,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:40:59,828 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:40:59,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:40:59,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:59,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:40:59,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:59,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:40:59,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:40:59,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/2626eb41648945bf9ce892e65476ea3a is 50, key is test_row_0/A:col10/1732779658749/Put/seqid=0 2024-11-28T07:40:59,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741954_1130 (size=12301) 2024-11-28T07:40:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:40:59,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. as already flushing 2024-11-28T07:40:59,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779719969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:40:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:40:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779719974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:40:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:40:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T07:41:00,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779720090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:00,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:00,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779720090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:00,247 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/2626eb41648945bf9ce892e65476ea3a 2024-11-28T07:41:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/6d0c9f6400cd4b09b29c2961b489c736 is 50, key is test_row_0/B:col10/1732779658749/Put/seqid=0 2024-11-28T07:41:00,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741955_1131 (size=12301) 2024-11-28T07:41:00,274 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/6d0c9f6400cd4b09b29c2961b489c736 2024-11-28T07:41:00,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/88ff30df3e504283b04219b0f0e46e2f is 50, key is test_row_0/C:col10/1732779658749/Put/seqid=0 2024-11-28T07:41:00,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56056 deadline: 1732779720300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:00,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56000 deadline: 1732779720301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:00,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:00,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741956_1132 (size=12301) 2024-11-28T07:41:00,321 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/88ff30df3e504283b04219b0f0e46e2f 2024-11-28T07:41:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/2626eb41648945bf9ce892e65476ea3a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/2626eb41648945bf9ce892e65476ea3a 2024-11-28T07:41:00,336 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/2626eb41648945bf9ce892e65476ea3a, entries=150, sequenceid=590, filesize=12.0 K 2024-11-28T07:41:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/6d0c9f6400cd4b09b29c2961b489c736 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/6d0c9f6400cd4b09b29c2961b489c736 2024-11-28T07:41:00,349 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/6d0c9f6400cd4b09b29c2961b489c736, entries=150, sequenceid=590, filesize=12.0 K 2024-11-28T07:41:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/88ff30df3e504283b04219b0f0e46e2f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/88ff30df3e504283b04219b0f0e46e2f 2024-11-28T07:41:00,359 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/88ff30df3e504283b04219b0f0e46e2f, entries=150, sequenceid=590, filesize=12.0 K 2024-11-28T07:41:00,363 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e36af693bfc3c757988aba93aff1bdda in 533ms, sequenceid=590, compaction requested=true 2024-11-28T07:41:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:41:00,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:41:00,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-28T07:41:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-28T07:41:00,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T07:41:00,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-28T07:41:00,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.4680 sec 2024-11-28T07:41:00,509 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-28T07:41:00,509 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,511 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-28T07:41:00,511 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,515 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,515 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,516 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,517 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-28T07:41:00,517 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,517 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,517 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,518 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,518 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,520 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-28T07:41:00,520 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,520 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,520 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:41:00,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:41:00,613 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-28T07:41:00,613 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,613 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:41:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:00,613 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:41:00,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:41:00,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:00,614 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-28T07:41:00,614 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,614 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,614 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5bfaafc2acda4b31b6772f069de61754 is 50, key is test_row_0/A:col10/1732779660610/Put/seqid=0 2024-11-28T07:41:00,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741957_1133 (size=12301) 2024-11-28T07:41:00,662 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-28T07:41:00,662 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,662 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,663 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,673 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-28T07:41:00,673 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,673 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,673 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-28T07:41:00,673 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,673 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:00,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:00,673 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,674 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:00,674 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:01,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5bfaafc2acda4b31b6772f069de61754 2024-11-28T07:41:01,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/14ec191b86634d3294c22431d7184c79 is 50, key is test_row_0/B:col10/1732779660610/Put/seqid=0 2024-11-28T07:41:01,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741958_1134 (size=12301) 2024-11-28T07:41:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T07:41:01,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:01,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2376 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2375 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1081 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3238 rows 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-28T07:41:01,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3243 rows 2024-11-28T07:41:01,052 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:41:01,052 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:41:01,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:01,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:01,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:01,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T07:41:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:01,067 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779661067"}]},"ts":"1732779661067"} 2024-11-28T07:41:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T07:41:01,070 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T07:41:01,070 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T07:41:01,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:41:01,077 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, UNASSIGN}] 2024-11-28T07:41:01,079 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, UNASSIGN 2024-11-28T07:41:01,081 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e36af693bfc3c757988aba93aff1bdda, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:01,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, UNASSIGN because future has completed 2024-11-28T07:41:01,083 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:41:01,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:41:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T07:41:01,242 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:41:01,242 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:41:01,242 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing e36af693bfc3c757988aba93aff1bdda, disabling compactions & flushes 2024-11-28T07:41:01,243 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:41:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T07:41:01,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/14ec191b86634d3294c22431d7184c79 2024-11-28T07:41:01,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/a048bc46dbe1456985ca5bfb00d7acc4 is 50, key is test_row_0/C:col10/1732779660610/Put/seqid=0 2024-11-28T07:41:01,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741959_1135 (size=12301) 2024-11-28T07:41:01,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/a048bc46dbe1456985ca5bfb00d7acc4 2024-11-28T07:41:01,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/5bfaafc2acda4b31b6772f069de61754 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5bfaafc2acda4b31b6772f069de61754 2024-11-28T07:41:01,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5bfaafc2acda4b31b6772f069de61754, entries=150, sequenceid=609, filesize=12.0 K 2024-11-28T07:41:01,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/14ec191b86634d3294c22431d7184c79 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/14ec191b86634d3294c22431d7184c79 2024-11-28T07:41:01,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/14ec191b86634d3294c22431d7184c79, entries=150, sequenceid=609, filesize=12.0 K 2024-11-28T07:41:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/a048bc46dbe1456985ca5bfb00d7acc4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a048bc46dbe1456985ca5bfb00d7acc4 2024-11-28T07:41:01,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a048bc46dbe1456985ca5bfb00d7acc4, entries=150, sequenceid=609, filesize=12.0 K 2024-11-28T07:41:01,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=20.13 KB/20610 for e36af693bfc3c757988aba93aff1bdda in 885ms, sequenceid=609, compaction requested=true 2024-11-28T07:41:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e36af693bfc3c757988aba93aff1bdda: 2024-11-28T07:41:01,498 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:41:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. after waiting 0 ms 2024-11-28T07:41:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:41:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:01,498 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. because compaction request was cancelled 2024-11-28T07:41:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e36af693bfc3c757988aba93aff1bdda:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:01,498 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. because compaction request was cancelled 2024-11-28T07:41:01,498 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing e36af693bfc3c757988aba93aff1bdda 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-28T07:41:01,498 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:A 2024-11-28T07:41:01,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:01,498 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:B 2024-11-28T07:41:01,498 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. because compaction request was cancelled 2024-11-28T07:41:01,498 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36af693bfc3c757988aba93aff1bdda:C 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=A 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=B 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e36af693bfc3c757988aba93aff1bdda, store=C 2024-11-28T07:41:01,498 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:01,503 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/8d2fe21ad1814149ac4e54e31fd5c2b5 is 50, key is test_row_0/A:col10/1732779660671/Put/seqid=0 2024-11-28T07:41:01,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741960_1136 (size=12301) 2024-11-28T07:41:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T07:41:01,907 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/8d2fe21ad1814149ac4e54e31fd5c2b5 2024-11-28T07:41:01,915 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/cf3ef8da6913417f8fb094f9cdfbc14d is 50, key is test_row_0/B:col10/1732779660671/Put/seqid=0 2024-11-28T07:41:01,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741961_1137 (size=12301) 2024-11-28T07:41:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T07:41:02,319 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/cf3ef8da6913417f8fb094f9cdfbc14d 2024-11-28T07:41:02,328 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/4bb04bd70de84bcf8c57d6d0531691a7 is 50, key is test_row_0/C:col10/1732779660671/Put/seqid=0 2024-11-28T07:41:02,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741962_1138 (size=12301) 2024-11-28T07:41:02,733 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=615 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/4bb04bd70de84bcf8c57d6d0531691a7 2024-11-28T07:41:02,739 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/A/8d2fe21ad1814149ac4e54e31fd5c2b5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/8d2fe21ad1814149ac4e54e31fd5c2b5 2024-11-28T07:41:02,745 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/8d2fe21ad1814149ac4e54e31fd5c2b5, entries=150, sequenceid=615, filesize=12.0 K 2024-11-28T07:41:02,746 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/B/cf3ef8da6913417f8fb094f9cdfbc14d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/cf3ef8da6913417f8fb094f9cdfbc14d 2024-11-28T07:41:02,752 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/cf3ef8da6913417f8fb094f9cdfbc14d, entries=150, sequenceid=615, filesize=12.0 K 2024-11-28T07:41:02,753 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/.tmp/C/4bb04bd70de84bcf8c57d6d0531691a7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4bb04bd70de84bcf8c57d6d0531691a7 2024-11-28T07:41:02,758 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4bb04bd70de84bcf8c57d6d0531691a7, entries=150, sequenceid=615, filesize=12.0 K 2024-11-28T07:41:02,759 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for e36af693bfc3c757988aba93aff1bdda in 1261ms, sequenceid=615, compaction requested=true 2024-11-28T07:41:02,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6671735b0b7f438088098027d1a70026, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92c7e7aeb3904f2ababab399b45c0709, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aca670d2a3414912a4c844c201b0666b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e3ab1c0aec7d42149d8cafc28c240031, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92b2ee598c1b45c1824ed6de46f8a0ec, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc38d681eaa245c98e00b04ed3c9c776, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/bb1ee655fca04fe9ab18c7d9a654ee18, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e594158a801145abace8bd2122f13e83, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/19d2288ec9ff407581737f591e731276, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/be6db0b9d16c48fbba64f07b98007790, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d334e6c0b57493db21d1b7838374dc5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/40a3547224aa4d358fef76fe14569ea2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/a82716fe10c443c791b17fabbf5e6bbd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43b4da80b915459096484433d0ccd914, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/f9f728acb5f04d6386c6a89b4dd58432, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6e8872a6c79d4932b37f48e4938036c2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d263b57751c430489bf8696c66f2541, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/928e7bd7efca4e8c90278eb92a803918, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/0242781c39e74bf29cf08a2ba75d2f33, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/050aa7e585804196a7465bdd20d2e354, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5b759e93292540e5966ea08222ccb9de, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5010fd50e46240598de23790970387bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/27c88feeca574c7ab56e77526c90906a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43d0457e60dd4cf1bbde136d9a36e36f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/507cc332a1644416bce9c0b022a130b0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/618d48ec8a1449b79f9afcc29838a209, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/051e7e351ee84dd6b4bd6ec1e3e9d994, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/ad3eddf319fe44e891222f9e65deb125, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc618fc9bba64ddeab7b17bc640271c7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d89c09e3ffd4665b4576a273495f1ff, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/16ab867bb84a4d0cb95aadeaaec9f0e7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/70a9e97e313f4a71bb2b9b231f6c7bfe, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/b1c1e7432e9744fe876c691f89cf36d9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d068e6804ef4ec5aa9da90c10938b3e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aec703b1d87b4338812d6c4b0a254a3f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/7c23c465a2174e5e87efb6e9dbc4ea25, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5e023d7a81314ed5bd9a9ec7d35f1e35] to archive 2024-11-28T07:41:02,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:41:02,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6671735b0b7f438088098027d1a70026 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6671735b0b7f438088098027d1a70026 2024-11-28T07:41:02,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92c7e7aeb3904f2ababab399b45c0709 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92c7e7aeb3904f2ababab399b45c0709 2024-11-28T07:41:02,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aca670d2a3414912a4c844c201b0666b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aca670d2a3414912a4c844c201b0666b 2024-11-28T07:41:02,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e3ab1c0aec7d42149d8cafc28c240031 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e3ab1c0aec7d42149d8cafc28c240031 2024-11-28T07:41:02,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92b2ee598c1b45c1824ed6de46f8a0ec to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/92b2ee598c1b45c1824ed6de46f8a0ec 2024-11-28T07:41:02,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc38d681eaa245c98e00b04ed3c9c776 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc38d681eaa245c98e00b04ed3c9c776 2024-11-28T07:41:02,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/bb1ee655fca04fe9ab18c7d9a654ee18 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/bb1ee655fca04fe9ab18c7d9a654ee18 2024-11-28T07:41:02,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e594158a801145abace8bd2122f13e83 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/e594158a801145abace8bd2122f13e83 2024-11-28T07:41:02,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/19d2288ec9ff407581737f591e731276 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/19d2288ec9ff407581737f591e731276 2024-11-28T07:41:02,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/be6db0b9d16c48fbba64f07b98007790 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/be6db0b9d16c48fbba64f07b98007790 2024-11-28T07:41:02,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d334e6c0b57493db21d1b7838374dc5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d334e6c0b57493db21d1b7838374dc5 2024-11-28T07:41:02,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/40a3547224aa4d358fef76fe14569ea2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/40a3547224aa4d358fef76fe14569ea2 2024-11-28T07:41:02,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/a82716fe10c443c791b17fabbf5e6bbd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/a82716fe10c443c791b17fabbf5e6bbd 2024-11-28T07:41:02,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43b4da80b915459096484433d0ccd914 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43b4da80b915459096484433d0ccd914 2024-11-28T07:41:02,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/f9f728acb5f04d6386c6a89b4dd58432 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/f9f728acb5f04d6386c6a89b4dd58432 2024-11-28T07:41:02,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6e8872a6c79d4932b37f48e4938036c2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6e8872a6c79d4932b37f48e4938036c2 2024-11-28T07:41:02,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d263b57751c430489bf8696c66f2541 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/6d263b57751c430489bf8696c66f2541 2024-11-28T07:41:02,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/928e7bd7efca4e8c90278eb92a803918 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/928e7bd7efca4e8c90278eb92a803918 2024-11-28T07:41:02,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/0242781c39e74bf29cf08a2ba75d2f33 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/0242781c39e74bf29cf08a2ba75d2f33 2024-11-28T07:41:02,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/050aa7e585804196a7465bdd20d2e354 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/050aa7e585804196a7465bdd20d2e354 2024-11-28T07:41:02,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5b759e93292540e5966ea08222ccb9de to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5b759e93292540e5966ea08222ccb9de 2024-11-28T07:41:02,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5010fd50e46240598de23790970387bf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5010fd50e46240598de23790970387bf 2024-11-28T07:41:02,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/27c88feeca574c7ab56e77526c90906a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/27c88feeca574c7ab56e77526c90906a 2024-11-28T07:41:02,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43d0457e60dd4cf1bbde136d9a36e36f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/43d0457e60dd4cf1bbde136d9a36e36f 2024-11-28T07:41:02,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/507cc332a1644416bce9c0b022a130b0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/507cc332a1644416bce9c0b022a130b0 2024-11-28T07:41:02,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/618d48ec8a1449b79f9afcc29838a209 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/618d48ec8a1449b79f9afcc29838a209 2024-11-28T07:41:02,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/051e7e351ee84dd6b4bd6ec1e3e9d994 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/051e7e351ee84dd6b4bd6ec1e3e9d994 2024-11-28T07:41:02,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/ad3eddf319fe44e891222f9e65deb125 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/ad3eddf319fe44e891222f9e65deb125 2024-11-28T07:41:02,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc618fc9bba64ddeab7b17bc640271c7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/dc618fc9bba64ddeab7b17bc640271c7 2024-11-28T07:41:02,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d89c09e3ffd4665b4576a273495f1ff to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d89c09e3ffd4665b4576a273495f1ff 2024-11-28T07:41:02,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/16ab867bb84a4d0cb95aadeaaec9f0e7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/16ab867bb84a4d0cb95aadeaaec9f0e7 2024-11-28T07:41:02,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/70a9e97e313f4a71bb2b9b231f6c7bfe to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/70a9e97e313f4a71bb2b9b231f6c7bfe 2024-11-28T07:41:02,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/b1c1e7432e9744fe876c691f89cf36d9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/b1c1e7432e9744fe876c691f89cf36d9 2024-11-28T07:41:02,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d068e6804ef4ec5aa9da90c10938b3e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/4d068e6804ef4ec5aa9da90c10938b3e 2024-11-28T07:41:02,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aec703b1d87b4338812d6c4b0a254a3f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/aec703b1d87b4338812d6c4b0a254a3f 2024-11-28T07:41:02,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/7c23c465a2174e5e87efb6e9dbc4ea25 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/7c23c465a2174e5e87efb6e9dbc4ea25 2024-11-28T07:41:02,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5e023d7a81314ed5bd9a9ec7d35f1e35 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5e023d7a81314ed5bd9a9ec7d35f1e35 2024-11-28T07:41:02,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2f93a950f50f4dd295572776f1f12582, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/377adeb654ea4e0cb403e6ce6b471d45, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/e4f236af8ea040308fbad3f6a78ac154, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/d75901216845467e8da20df1c1a12dac, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/24c2ba3d63534804bcc3d726c300a57f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/9a0c51aa58cc4e9d8e52389d8dd75ece, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c2b1cca29e3346f2b72858011348074c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/608339954b274b88bfdf527a0459cb01, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/20e0b868d59e4b6e8181d1e49d6d5341, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2cb613b9dc8e428daa3917b9e2641f55, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0096babbe7c144629d8e253c4e57738f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/dcb75d8c72df4587ab53703dda651d0a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4793dc2e24ef406daaf8140a79dce23e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/065436bee16d47fa8f0347f580ef63e9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/ca0da42f8fc44575a6d5bc5c1f597eed, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/3415f5ed405d4552b84776ac3ad14364, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5d894a20ad604518a97af8150e90d94b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/b10974046d834b09b2b1573df66b23d4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c5cba1e208204e5e8978915f40e4615f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1bece14f350a46b8b8e1711c28c19bba, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1b950c2786c34a7e86a9475f9680cdee, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/93930e6e7fa54b6195b868b0d20a377e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/414c8fba61184e50954343a782f75495, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5f30f3193fbf40f9adccd13044bba52c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4f97edbb46f9493db6bb11c7a2cde192, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0a54e3b717a24eb8a588ffb97cb4cf17, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/26e7a88b5b564c84a7528bad70dbb435, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0239b731628f40719c929e58ca83d3b0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/059519378a344d3284d2ac1156c61cc7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1621421365484d418d5047c796a907bc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/f680785383a0456f9d39439c4bfc74d7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1ff6b09535ef48ceab623a360e63fb25, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/8ace933579564b2e9fadce2254729e8e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5c50655f44514c3ba6702c1997d4421e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c4e4eb9ebb3645cea65da40d9e3f8535, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2ff0638a5be04345bd7831931449537d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/03fb16e8069847bf8bea7e180d25ac97] to archive 2024-11-28T07:41:02,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:41:02,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2f93a950f50f4dd295572776f1f12582 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2f93a950f50f4dd295572776f1f12582 2024-11-28T07:41:02,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/377adeb654ea4e0cb403e6ce6b471d45 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/377adeb654ea4e0cb403e6ce6b471d45 2024-11-28T07:41:02,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/e4f236af8ea040308fbad3f6a78ac154 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/e4f236af8ea040308fbad3f6a78ac154 2024-11-28T07:41:02,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/d75901216845467e8da20df1c1a12dac to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/d75901216845467e8da20df1c1a12dac 2024-11-28T07:41:02,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/24c2ba3d63534804bcc3d726c300a57f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/24c2ba3d63534804bcc3d726c300a57f 2024-11-28T07:41:02,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/9a0c51aa58cc4e9d8e52389d8dd75ece to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/9a0c51aa58cc4e9d8e52389d8dd75ece 2024-11-28T07:41:02,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c2b1cca29e3346f2b72858011348074c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c2b1cca29e3346f2b72858011348074c 2024-11-28T07:41:02,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/608339954b274b88bfdf527a0459cb01 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/608339954b274b88bfdf527a0459cb01 2024-11-28T07:41:02,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/20e0b868d59e4b6e8181d1e49d6d5341 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/20e0b868d59e4b6e8181d1e49d6d5341 2024-11-28T07:41:02,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2cb613b9dc8e428daa3917b9e2641f55 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2cb613b9dc8e428daa3917b9e2641f55 2024-11-28T07:41:02,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0096babbe7c144629d8e253c4e57738f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0096babbe7c144629d8e253c4e57738f 2024-11-28T07:41:02,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/dcb75d8c72df4587ab53703dda651d0a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/dcb75d8c72df4587ab53703dda651d0a 2024-11-28T07:41:02,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4793dc2e24ef406daaf8140a79dce23e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4793dc2e24ef406daaf8140a79dce23e 2024-11-28T07:41:02,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/065436bee16d47fa8f0347f580ef63e9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/065436bee16d47fa8f0347f580ef63e9 2024-11-28T07:41:02,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/ca0da42f8fc44575a6d5bc5c1f597eed to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/ca0da42f8fc44575a6d5bc5c1f597eed 2024-11-28T07:41:02,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/3415f5ed405d4552b84776ac3ad14364 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/3415f5ed405d4552b84776ac3ad14364 2024-11-28T07:41:02,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5d894a20ad604518a97af8150e90d94b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5d894a20ad604518a97af8150e90d94b 2024-11-28T07:41:02,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/b10974046d834b09b2b1573df66b23d4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/b10974046d834b09b2b1573df66b23d4 2024-11-28T07:41:02,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c5cba1e208204e5e8978915f40e4615f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c5cba1e208204e5e8978915f40e4615f 2024-11-28T07:41:02,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1bece14f350a46b8b8e1711c28c19bba to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1bece14f350a46b8b8e1711c28c19bba 2024-11-28T07:41:02,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1b950c2786c34a7e86a9475f9680cdee to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1b950c2786c34a7e86a9475f9680cdee 2024-11-28T07:41:02,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/93930e6e7fa54b6195b868b0d20a377e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/93930e6e7fa54b6195b868b0d20a377e 2024-11-28T07:41:02,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/414c8fba61184e50954343a782f75495 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/414c8fba61184e50954343a782f75495 2024-11-28T07:41:02,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5f30f3193fbf40f9adccd13044bba52c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5f30f3193fbf40f9adccd13044bba52c 2024-11-28T07:41:02,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4f97edbb46f9493db6bb11c7a2cde192 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/4f97edbb46f9493db6bb11c7a2cde192 2024-11-28T07:41:02,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0a54e3b717a24eb8a588ffb97cb4cf17 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0a54e3b717a24eb8a588ffb97cb4cf17 2024-11-28T07:41:02,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/26e7a88b5b564c84a7528bad70dbb435 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/26e7a88b5b564c84a7528bad70dbb435 2024-11-28T07:41:02,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0239b731628f40719c929e58ca83d3b0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/0239b731628f40719c929e58ca83d3b0 2024-11-28T07:41:02,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/059519378a344d3284d2ac1156c61cc7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/059519378a344d3284d2ac1156c61cc7 2024-11-28T07:41:02,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1621421365484d418d5047c796a907bc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1621421365484d418d5047c796a907bc 2024-11-28T07:41:02,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/f680785383a0456f9d39439c4bfc74d7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/f680785383a0456f9d39439c4bfc74d7 2024-11-28T07:41:02,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1ff6b09535ef48ceab623a360e63fb25 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/1ff6b09535ef48ceab623a360e63fb25 2024-11-28T07:41:02,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/8ace933579564b2e9fadce2254729e8e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/8ace933579564b2e9fadce2254729e8e 2024-11-28T07:41:02,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5c50655f44514c3ba6702c1997d4421e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/5c50655f44514c3ba6702c1997d4421e 2024-11-28T07:41:02,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c4e4eb9ebb3645cea65da40d9e3f8535 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/c4e4eb9ebb3645cea65da40d9e3f8535 2024-11-28T07:41:02,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2ff0638a5be04345bd7831931449537d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/2ff0638a5be04345bd7831931449537d 2024-11-28T07:41:02,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/03fb16e8069847bf8bea7e180d25ac97 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/03fb16e8069847bf8bea7e180d25ac97 2024-11-28T07:41:02,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c3620e9728f4ed89dc2dcc4ee6c8275, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/8ebedf8350e44067850d4e0d53ca7973, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/511ab0dffeb44e16987574ffe8c76674, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9a1ddf486daf45898cc121d3bc709a74, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b82be6c773c348edb121d3b52f4d39c3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c200b99b0764ac88afbd047acbcb7d0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/51057362353e49c4ae19b6fbce700e43, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/628f2710c8044e1fbda00950679a943a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2b6d3c30039f4f59a33d81fb1e9a47a9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/f82dbf86c9994880ad9c378a87236852, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a39c254862474d019d1b8e49f46d0d69, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/80a8ebf711d249689e74bf5752a13d33, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3c2c619660634a99bd2804e34b1c7538, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9be585f57ff44111ac354a48d89f424f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1b08af4b5f2c46c5a4853165b326bd3d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/48515b1323404d1eba23c4beca47c37b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4d60b8df69974e6185cedcff33121f3c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/30f1a456a75140e0acc922922288c694, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9734713a59ba4a62a5918decbd8b0ec6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3699d1ae1c9b417aac6783c9b01e1166, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/6bb1fe4fa52443958b4eadd217c3638e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/da26dbeaee694acab2126520a99ec8e2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/e158ea52a9be41409ccaa50388935bbe, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9705481b8ddf476ba41e12d3d0baafdc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ec7ae4fe85b9454cb428404b4bee2157, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/78cd1e50c41a42f0ac45be0e6b8fe965, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c5ac50ec50164f3796e2c352894ee2b8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/43033f17de7049c3ba332b622d42b2cc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b29365e7b9804aa2924d9c2fc4cdc2ba, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/5e0a7409cee344abab5a26718403093a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9ccf230097ad4d54869cf4bdbe73cf8f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ef61014866f943438397564ddf22f0f0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1da47a4c414449cb8b03a3b85fcc729c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3af0497c7e6c45d387c9b416fa2e67ca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/0ca973a83d494dd48400ef5b80343476, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2822a28913b1484d898ae6726eebfebd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/418f6862921b4162a4371d3b3ade5b87] to archive 2024-11-28T07:41:02,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:41:02,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c3620e9728f4ed89dc2dcc4ee6c8275 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c3620e9728f4ed89dc2dcc4ee6c8275 2024-11-28T07:41:02,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/8ebedf8350e44067850d4e0d53ca7973 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/8ebedf8350e44067850d4e0d53ca7973 2024-11-28T07:41:02,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/511ab0dffeb44e16987574ffe8c76674 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/511ab0dffeb44e16987574ffe8c76674 2024-11-28T07:41:02,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9a1ddf486daf45898cc121d3bc709a74 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9a1ddf486daf45898cc121d3bc709a74 2024-11-28T07:41:02,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b82be6c773c348edb121d3b52f4d39c3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b82be6c773c348edb121d3b52f4d39c3 2024-11-28T07:41:02,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c200b99b0764ac88afbd047acbcb7d0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1c200b99b0764ac88afbd047acbcb7d0 2024-11-28T07:41:02,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/51057362353e49c4ae19b6fbce700e43 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/51057362353e49c4ae19b6fbce700e43 2024-11-28T07:41:02,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/628f2710c8044e1fbda00950679a943a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/628f2710c8044e1fbda00950679a943a 2024-11-28T07:41:02,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2b6d3c30039f4f59a33d81fb1e9a47a9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2b6d3c30039f4f59a33d81fb1e9a47a9 2024-11-28T07:41:02,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/f82dbf86c9994880ad9c378a87236852 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/f82dbf86c9994880ad9c378a87236852 2024-11-28T07:41:02,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a39c254862474d019d1b8e49f46d0d69 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a39c254862474d019d1b8e49f46d0d69 2024-11-28T07:41:02,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/80a8ebf711d249689e74bf5752a13d33 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/80a8ebf711d249689e74bf5752a13d33 2024-11-28T07:41:02,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3c2c619660634a99bd2804e34b1c7538 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3c2c619660634a99bd2804e34b1c7538 2024-11-28T07:41:02,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9be585f57ff44111ac354a48d89f424f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9be585f57ff44111ac354a48d89f424f 2024-11-28T07:41:02,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1b08af4b5f2c46c5a4853165b326bd3d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1b08af4b5f2c46c5a4853165b326bd3d 2024-11-28T07:41:02,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/48515b1323404d1eba23c4beca47c37b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/48515b1323404d1eba23c4beca47c37b 2024-11-28T07:41:02,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4d60b8df69974e6185cedcff33121f3c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4d60b8df69974e6185cedcff33121f3c 2024-11-28T07:41:02,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/30f1a456a75140e0acc922922288c694 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/30f1a456a75140e0acc922922288c694 2024-11-28T07:41:02,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9734713a59ba4a62a5918decbd8b0ec6 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9734713a59ba4a62a5918decbd8b0ec6 2024-11-28T07:41:02,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3699d1ae1c9b417aac6783c9b01e1166 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3699d1ae1c9b417aac6783c9b01e1166 2024-11-28T07:41:02,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/6bb1fe4fa52443958b4eadd217c3638e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/6bb1fe4fa52443958b4eadd217c3638e 2024-11-28T07:41:02,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/da26dbeaee694acab2126520a99ec8e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/da26dbeaee694acab2126520a99ec8e2 2024-11-28T07:41:02,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/e158ea52a9be41409ccaa50388935bbe to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/e158ea52a9be41409ccaa50388935bbe 2024-11-28T07:41:02,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9705481b8ddf476ba41e12d3d0baafdc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9705481b8ddf476ba41e12d3d0baafdc 2024-11-28T07:41:02,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ec7ae4fe85b9454cb428404b4bee2157 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ec7ae4fe85b9454cb428404b4bee2157 2024-11-28T07:41:02,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/78cd1e50c41a42f0ac45be0e6b8fe965 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/78cd1e50c41a42f0ac45be0e6b8fe965 2024-11-28T07:41:02,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c5ac50ec50164f3796e2c352894ee2b8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c5ac50ec50164f3796e2c352894ee2b8 2024-11-28T07:41:02,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/43033f17de7049c3ba332b622d42b2cc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/43033f17de7049c3ba332b622d42b2cc 2024-11-28T07:41:02,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b29365e7b9804aa2924d9c2fc4cdc2ba to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b29365e7b9804aa2924d9c2fc4cdc2ba 2024-11-28T07:41:02,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/5e0a7409cee344abab5a26718403093a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/5e0a7409cee344abab5a26718403093a 2024-11-28T07:41:02,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9ccf230097ad4d54869cf4bdbe73cf8f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/9ccf230097ad4d54869cf4bdbe73cf8f 2024-11-28T07:41:02,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ef61014866f943438397564ddf22f0f0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/ef61014866f943438397564ddf22f0f0 2024-11-28T07:41:02,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1da47a4c414449cb8b03a3b85fcc729c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/1da47a4c414449cb8b03a3b85fcc729c 2024-11-28T07:41:02,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3af0497c7e6c45d387c9b416fa2e67ca to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/3af0497c7e6c45d387c9b416fa2e67ca 2024-11-28T07:41:02,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/0ca973a83d494dd48400ef5b80343476 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/0ca973a83d494dd48400ef5b80343476 2024-11-28T07:41:02,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2822a28913b1484d898ae6726eebfebd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/2822a28913b1484d898ae6726eebfebd 2024-11-28T07:41:02,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/418f6862921b4162a4371d3b3ade5b87 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/418f6862921b4162a4371d3b3ade5b87 2024-11-28T07:41:02,947 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/recovered.edits/618.seqid, newMaxSeqId=618, maxSeqId=1 2024-11-28T07:41:02,948 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. 2024-11-28T07:41:02,949 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for e36af693bfc3c757988aba93aff1bdda: Waiting for close lock at 1732779661242Running coprocessor pre-close hooks at 1732779661242Disabling compacts and flushes for region at 1732779661242Disabling writes for close at 1732779661498 (+256 ms)Obtaining lock to block concurrent updates at 1732779661498Preparing flush snapshotting stores in e36af693bfc3c757988aba93aff1bdda at 1732779661498Finished memstore snapshotting TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732779661498Flushing stores of TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda. at 1732779661499 (+1 ms)Flushing e36af693bfc3c757988aba93aff1bdda/A: creating writer at 1732779661499Flushing e36af693bfc3c757988aba93aff1bdda/A: appending metadata at 1732779661502 (+3 ms)Flushing e36af693bfc3c757988aba93aff1bdda/A: closing flushed file at 1732779661502Flushing e36af693bfc3c757988aba93aff1bdda/B: creating writer at 1732779661912 (+410 ms)Flushing e36af693bfc3c757988aba93aff1bdda/B: appending metadata at 1732779661914 (+2 ms)Flushing e36af693bfc3c757988aba93aff1bdda/B: closing flushed file at 1732779661914Flushing e36af693bfc3c757988aba93aff1bdda/C: creating writer at 1732779662324 (+410 ms)Flushing e36af693bfc3c757988aba93aff1bdda/C: appending metadata at 1732779662328 (+4 ms)Flushing e36af693bfc3c757988aba93aff1bdda/C: closing flushed file at 1732779662328Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26736932: reopening flushed file at 1732779662738 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49e876a9: reopening flushed file at 1732779662745 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@662a9a0c: reopening flushed file at 1732779662752 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for e36af693bfc3c757988aba93aff1bdda in 1261ms, sequenceid=615, compaction requested=true at 1732779662759 (+7 ms)Writing region close event to WAL at 1732779662944 (+185 ms)Running coprocessor post-close hooks at 1732779662948 (+4 ms)Closed at 1732779662948 2024-11-28T07:41:02,951 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:41:02,952 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e36af693bfc3c757988aba93aff1bdda, regionState=CLOSED 2024-11-28T07:41:02,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:41:02,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-28T07:41:02,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure e36af693bfc3c757988aba93aff1bdda, server=f64cf8bdf19b,34437,1732779635619 in 1.8710 sec 2024-11-28T07:41:02,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-28T07:41:02,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e36af693bfc3c757988aba93aff1bdda, UNASSIGN in 1.8800 sec 2024-11-28T07:41:02,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-28T07:41:02,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8890 sec 2024-11-28T07:41:02,963 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779662963"}]},"ts":"1732779662963"} 2024-11-28T07:41:02,964 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T07:41:02,964 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T07:41:02,967 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9050 sec 2024-11-28T07:41:03,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-28T07:41:03,221 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:03,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T07:41:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:03,230 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:03,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T07:41:03,232 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:03,240 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:41:03,252 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/recovered.edits] 2024-11-28T07:41:03,260 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/2626eb41648945bf9ce892e65476ea3a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/2626eb41648945bf9ce892e65476ea3a 2024-11-28T07:41:03,262 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5bfaafc2acda4b31b6772f069de61754 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/5bfaafc2acda4b31b6772f069de61754 2024-11-28T07:41:03,264 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/8d2fe21ad1814149ac4e54e31fd5c2b5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/8d2fe21ad1814149ac4e54e31fd5c2b5 2024-11-28T07:41:03,266 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/cab75ca7067e4e9aa4d801f8a48fa4d0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/cab75ca7067e4e9aa4d801f8a48fa4d0 2024-11-28T07:41:03,267 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/da2074e75f36434f860aaebfbf577357 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/A/da2074e75f36434f860aaebfbf577357 2024-11-28T07:41:03,271 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/14ec191b86634d3294c22431d7184c79 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/14ec191b86634d3294c22431d7184c79 2024-11-28T07:41:03,272 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/62269353f8ad47c08f94a9c9e96ae956 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/62269353f8ad47c08f94a9c9e96ae956 2024-11-28T07:41:03,274 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/6d0c9f6400cd4b09b29c2961b489c736 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/6d0c9f6400cd4b09b29c2961b489c736 2024-11-28T07:41:03,276 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/77167bb3dd844d18994bf12be1d38697 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/77167bb3dd844d18994bf12be1d38697 2024-11-28T07:41:03,280 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/cf3ef8da6913417f8fb094f9cdfbc14d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/B/cf3ef8da6913417f8fb094f9cdfbc14d 2024-11-28T07:41:03,283 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4bb04bd70de84bcf8c57d6d0531691a7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/4bb04bd70de84bcf8c57d6d0531691a7 2024-11-28T07:41:03,285 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/88ff30df3e504283b04219b0f0e46e2f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/88ff30df3e504283b04219b0f0e46e2f 2024-11-28T07:41:03,286 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a048bc46dbe1456985ca5bfb00d7acc4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/a048bc46dbe1456985ca5bfb00d7acc4 2024-11-28T07:41:03,288 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b0537145519d4d00a6875150f7ec2998 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/b0537145519d4d00a6875150f7ec2998 2024-11-28T07:41:03,289 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c405b6591dfb4bdcb49e58f8258e5f51 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/C/c405b6591dfb4bdcb49e58f8258e5f51 2024-11-28T07:41:03,293 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/recovered.edits/618.seqid to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda/recovered.edits/618.seqid 2024-11-28T07:41:03,294 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/e36af693bfc3c757988aba93aff1bdda 2024-11-28T07:41:03,294 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T07:41:03,300 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:03,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-28T07:41:03,314 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T07:41:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T07:41:03,356 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T07:41:03,358 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:03,358 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T07:41:03,358 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732779663358"}]},"ts":"9223372036854775807"} 2024-11-28T07:41:03,363 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T07:41:03,363 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e36af693bfc3c757988aba93aff1bdda, NAME => 'TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T07:41:03,363 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T07:41:03,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732779663363"}]},"ts":"9223372036854775807"} 2024-11-28T07:41:03,367 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T07:41:03,367 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:03,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 142 msec 2024-11-28T07:41:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T07:41:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T07:41:03,552 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:03,567 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f64cf8bdf19b:34437-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2099947969_22 at /127.0.0.1:40394 [Waiting for operation #411] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1186479210_22 at /127.0.0.1:33904 [Waiting for operation #383] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 277) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7170 (was 7597) 2024-11-28T07:41:03,579 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=369, ProcessCount=11, AvailableMemoryMB=7169 2024-11-28T07:41:03,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:41:03,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:41:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:03,584 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:41:03,584 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:03,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-28T07:41:03,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T07:41:03,585 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:41:03,593 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T07:41:03,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741963_1139 (size=1061) 2024-11-28T07:41:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T07:41:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T07:41:04,000 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:41:04,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741964_1140 (size=53) 2024-11-28T07:41:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T07:41:04,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:41:04,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4e9a306a7a5304212409c030a7b68ce0, disabling compactions & flushes 2024-11-28T07:41:04,412 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. after waiting 0 ms 2024-11-28T07:41:04,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,412 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,412 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e9a306a7a5304212409c030a7b68ce0: Waiting for close lock at 1732779664412Disabling compacts and flushes for region at 1732779664412Disabling writes for close at 1732779664412Writing region close event to WAL at 1732779664412Closed at 1732779664412 2024-11-28T07:41:04,414 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:41:04,414 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732779664414"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779664414"}]},"ts":"1732779664414"} 2024-11-28T07:41:04,416 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:41:04,417 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:41:04,418 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779664417"}]},"ts":"1732779664417"} 2024-11-28T07:41:04,419 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T07:41:04,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, ASSIGN}] 2024-11-28T07:41:04,420 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, ASSIGN 2024-11-28T07:41:04,421 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=false 2024-11-28T07:41:04,572 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:04,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, ASSIGN because future has completed 2024-11-28T07:41:04,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:41:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T07:41:04,729 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,729 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:41:04,730 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,730 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:41:04,730 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,730 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,731 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,733 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:04,733 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e9a306a7a5304212409c030a7b68ce0 columnFamilyName A 2024-11-28T07:41:04,734 DEBUG [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:04,734 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(327): Store=4e9a306a7a5304212409c030a7b68ce0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:04,734 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,735 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:04,736 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e9a306a7a5304212409c030a7b68ce0 columnFamilyName B 2024-11-28T07:41:04,736 DEBUG [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:04,736 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(327): Store=4e9a306a7a5304212409c030a7b68ce0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:04,736 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,737 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:04,737 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e9a306a7a5304212409c030a7b68ce0 columnFamilyName C 2024-11-28T07:41:04,738 DEBUG [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:04,738 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(327): Store=4e9a306a7a5304212409c030a7b68ce0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:04,738 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,738 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,739 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,739 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,740 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,741 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,741 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:41:04,742 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,744 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:41:04,744 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 4e9a306a7a5304212409c030a7b68ce0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63453214, jitterRate=-0.054473429918289185}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:41:04,744 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:04,745 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 4e9a306a7a5304212409c030a7b68ce0: Running coprocessor pre-open hook at 1732779664730Writing region info on filesystem at 1732779664730Initializing all the Stores at 1732779664731 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779664731Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779664731Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779664731Cleaning up temporary data from old regions at 1732779664741 (+10 ms)Running coprocessor post-open hooks at 1732779664744 (+3 ms)Region opened successfully at 1732779664745 (+1 ms) 2024-11-28T07:41:04,746 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., pid=34, masterSystemTime=1732779664726 2024-11-28T07:41:04,748 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,748 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:04,749 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:04,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:41:04,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-28T07:41:04,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 in 177 msec 2024-11-28T07:41:04,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-28T07:41:04,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, ASSIGN in 334 msec 2024-11-28T07:41:04,756 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:41:04,756 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779664756"}]},"ts":"1732779664756"} 2024-11-28T07:41:04,757 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T07:41:04,758 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:41:04,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-28T07:41:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-28T07:41:05,730 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:05,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:05,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:05,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:05,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b44f32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:05,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:05,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c68170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:05,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:05,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:05,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:05,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:41:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,39063,1732779634677 2024-11-28T07:41:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3accb36a 2024-11-28T07:41:05,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:41:05,742 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:41:05,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:41:05,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:41:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:05,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741965_1141 (size=1097) 2024-11-28T07:41:06,166 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T07:41:06,166 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T07:41:06,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:41:06,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, REOPEN/MOVE}] 2024-11-28T07:41:06,178 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, REOPEN/MOVE 2024-11-28T07:41:06,178 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:06,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, REOPEN/MOVE because future has completed 2024-11-28T07:41:06,180 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:41:06,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:41:06,333 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,333 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:41:06,333 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 4e9a306a7a5304212409c030a7b68ce0, disabling compactions & flushes 2024-11-28T07:41:06,333 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,333 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,333 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. after waiting 0 ms 2024-11-28T07:41:06,333 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,337 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:41:06,338 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,338 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 4e9a306a7a5304212409c030a7b68ce0: Waiting for close lock at 1732779666333Running coprocessor pre-close hooks at 1732779666333Disabling compacts and flushes for region at 1732779666333Disabling writes for close at 1732779666333Writing region close event to WAL at 1732779666334 (+1 ms)Running coprocessor post-close hooks at 1732779666338 (+4 ms)Closed at 1732779666338 2024-11-28T07:41:06,338 WARN [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 4e9a306a7a5304212409c030a7b68ce0 to self. 2024-11-28T07:41:06,340 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,341 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=CLOSED 2024-11-28T07:41:06,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:41:06,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-28T07:41:06,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 in 164 msec 2024-11-28T07:41:06,347 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, REOPEN/MOVE; state=CLOSED, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=true 2024-11-28T07:41:06,498 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:06,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, REOPEN/MOVE because future has completed 2024-11-28T07:41:06,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:41:06,660 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,660 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:41:06,662 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,662 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:41:06,662 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,662 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,667 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,668 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:06,676 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e9a306a7a5304212409c030a7b68ce0 columnFamilyName A 2024-11-28T07:41:06,692 DEBUG [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:06,693 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(327): Store=4e9a306a7a5304212409c030a7b68ce0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:06,694 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,696 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:06,696 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e9a306a7a5304212409c030a7b68ce0 columnFamilyName B 2024-11-28T07:41:06,696 DEBUG [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:06,697 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(327): Store=4e9a306a7a5304212409c030a7b68ce0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:06,697 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,698 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:06,698 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e9a306a7a5304212409c030a7b68ce0 columnFamilyName C 2024-11-28T07:41:06,698 DEBUG [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:06,700 INFO [StoreOpener-4e9a306a7a5304212409c030a7b68ce0-1 {}] regionserver.HStore(327): Store=4e9a306a7a5304212409c030a7b68ce0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:06,700 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,700 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,701 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,702 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,704 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,704 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,705 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:41:06,716 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,718 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 4e9a306a7a5304212409c030a7b68ce0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61560602, jitterRate=-0.08267554640769958}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:41:06,718 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,719 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 4e9a306a7a5304212409c030a7b68ce0: Running coprocessor pre-open hook at 1732779666662Writing region info on filesystem at 1732779666662Initializing all the Stores at 1732779666663 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779666663Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779666663Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779666663Cleaning up temporary data from old regions at 1732779666704 (+41 ms)Running coprocessor post-open hooks at 1732779666718 (+14 ms)Region opened successfully at 1732779666719 (+1 ms) 2024-11-28T07:41:06,720 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., pid=39, masterSystemTime=1732779666654 2024-11-28T07:41:06,724 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,724 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=OPEN, openSeqNum=5, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:06,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:41:06,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-28T07:41:06,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 in 230 msec 2024-11-28T07:41:06,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-28T07:41:06,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, REOPEN/MOVE in 556 msec 2024-11-28T07:41:06,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-28T07:41:06,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 569 msec 2024-11-28T07:41:06,743 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 992 msec 2024-11-28T07:41:06,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-28T07:41:06,745 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:06,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea11d07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,752 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289e5f6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,766 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0f283e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,767 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,768 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,769 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cee228, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,773 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a99cc77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,775 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728f3b52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,780 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5148995e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,783 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539ae260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,790 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9beb18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,792 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478870e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ee16cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,796 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e9c9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,801 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2559f6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341602f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,814 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa9c030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,817 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7829860b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,821 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,821 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:06,821 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:06,823 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e68928c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:06,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:06,826 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:06,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:06,828 DEBUG [Thread-705 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:06,830 DEBUG [Thread-705 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,830 DEBUG [Thread-705 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-28T07:41:06,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T07:41:06,833 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:06,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:06,837 DEBUG [Thread-698 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,840 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,841 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,844 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,845 DEBUG [Thread-696 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,847 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,857 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,877 DEBUG [Thread-702 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,897 DEBUG [Thread-694 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:06,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:41:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:06,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:06,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,909 DEBUG [Thread-700 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:06,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:06,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:06,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:41:06,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T07:41:06,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T07:41:06,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:06,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:06,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:06,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:06,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779666956, max timestamp=1732779666986Num uniques 859; 2024-11-28T07:41:06,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:06,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779666956, max timestamp=1732779666986Num uniques 859; 2024-11-28T07:41:06,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779666956, max timestamp=1732779666986Num uniques 859; 2024-11-28T07:41:06,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:06,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:07,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112883a937f73908474db57646ccd9e76b97_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779666888/Put/seqid=0 2024-11-28T07:41:07,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741966_1142 (size=19474) 2024-11-28T07:41:07,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:07,033 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112883a937f73908474db57646ccd9e76b97_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883a937f73908474db57646ccd9e76b97_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:07,034 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/555ee48a099d431c96f26ae2aff22aca, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:07,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/555ee48a099d431c96f26ae2aff22aca is 175, key is test_row_0/A:col10/1732779666888/Put/seqid=0 2024-11-28T07:41:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779727034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779727035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779727037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741967_1143 (size=56733) 2024-11-28T07:41:07,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779727048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779727049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,078 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/555ee48a099d431c96f26ae2aff22aca 2024-11-28T07:41:07,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/6c51071ef74246cc9812ed892f0ff07d is 50, key is test_row_0/B:col10/1732779666888/Put/seqid=0 2024-11-28T07:41:07,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741968_1144 (size=12001) 2024-11-28T07:41:07,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T07:41:07,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T07:41:07,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779727160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779727161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779727161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779727177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779727178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T07:41:07,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:07,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779727371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779727371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779727371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779727396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779727396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T07:41:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T07:41:07,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/6c51071ef74246cc9812ed892f0ff07d 2024-11-28T07:41:07,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T07:41:07,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:07,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:07,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/31ae9951fed4481ebf97682afd2ecabf is 50, key is test_row_0/C:col10/1732779666888/Put/seqid=0 2024-11-28T07:41:07,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741969_1145 (size=12001) 2024-11-28T07:41:07,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/31ae9951fed4481ebf97682afd2ecabf 2024-11-28T07:41:07,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779727680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779727681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779727681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/555ee48a099d431c96f26ae2aff22aca as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/555ee48a099d431c96f26ae2aff22aca 2024-11-28T07:41:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/555ee48a099d431c96f26ae2aff22aca, entries=300, sequenceid=15, filesize=55.4 K 2024-11-28T07:41:07,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/6c51071ef74246cc9812ed892f0ff07d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/6c51071ef74246cc9812ed892f0ff07d 2024-11-28T07:41:07,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/6c51071ef74246cc9812ed892f0ff07d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T07:41:07,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/31ae9951fed4481ebf97682afd2ecabf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/31ae9951fed4481ebf97682afd2ecabf 2024-11-28T07:41:07,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779727700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779727700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:07,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/31ae9951fed4481ebf97682afd2ecabf, entries=150, sequenceid=15, filesize=11.7 K 2024-11-28T07:41:07,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4e9a306a7a5304212409c030a7b68ce0 in 826ms, sequenceid=15, compaction requested=false 2024-11-28T07:41:07,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:07,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-28T07:41:07,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:07,763 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T07:41:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:07,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:07,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128964434f71a80492e8cca1b37952f5d3a_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779667046/Put/seqid=0 2024-11-28T07:41:07,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741970_1146 (size=12154) 2024-11-28T07:41:07,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:07,839 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128964434f71a80492e8cca1b37952f5d3a_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128964434f71a80492e8cca1b37952f5d3a_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/59a3d52722014cc78cc73b2ffd543ef2, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:07,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/59a3d52722014cc78cc73b2ffd543ef2 is 175, key is test_row_0/A:col10/1732779667046/Put/seqid=0 2024-11-28T07:41:07,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741971_1147 (size=30955) 2024-11-28T07:41:07,891 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/59a3d52722014cc78cc73b2ffd543ef2 2024-11-28T07:41:07,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/dfeace86b5d9400ab85a6b722eb3fe67 is 50, key is test_row_0/B:col10/1732779667046/Put/seqid=0 2024-11-28T07:41:07,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741972_1148 (size=12001) 2024-11-28T07:41:07,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T07:41:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:08,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:08,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779728216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779728220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779728220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779728222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779728231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779728331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779728331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779728340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,364 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/dfeace86b5d9400ab85a6b722eb3fe67 2024-11-28T07:41:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/3102c0aeaf734fd18c573c5a37cf899b is 50, key is test_row_0/C:col10/1732779667046/Put/seqid=0 2024-11-28T07:41:08,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741973_1149 (size=12001) 2024-11-28T07:41:08,396 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/3102c0aeaf734fd18c573c5a37cf899b 2024-11-28T07:41:08,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/59a3d52722014cc78cc73b2ffd543ef2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/59a3d52722014cc78cc73b2ffd543ef2 2024-11-28T07:41:08,416 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/59a3d52722014cc78cc73b2ffd543ef2, entries=150, sequenceid=51, filesize=30.2 K 2024-11-28T07:41:08,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/dfeace86b5d9400ab85a6b722eb3fe67 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dfeace86b5d9400ab85a6b722eb3fe67 2024-11-28T07:41:08,424 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dfeace86b5d9400ab85a6b722eb3fe67, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T07:41:08,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/3102c0aeaf734fd18c573c5a37cf899b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/3102c0aeaf734fd18c573c5a37cf899b 2024-11-28T07:41:08,443 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/3102c0aeaf734fd18c573c5a37cf899b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-28T07:41:08,445 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4e9a306a7a5304212409c030a7b68ce0 in 681ms, sequenceid=51, compaction requested=false 2024-11-28T07:41:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-28T07:41:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-28T07:41:08,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-28T07:41:08,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-28T07:41:08,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6240 sec 2024-11-28T07:41:08,532 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:41:08,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:41:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:08,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f266ae8b20d840b2b17a2b5ce1a2d110_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779668555/Put/seqid=0 2024-11-28T07:41:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779728625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779728628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741974_1150 (size=14594) 2024-11-28T07:41:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779728630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779728740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779728740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779728751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779728951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779728951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779728960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:08,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-28T07:41:08,992 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:08,992 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-28T07:41:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T07:41:08,997 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:08,998 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:08,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:09,040 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:09,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f266ae8b20d840b2b17a2b5ce1a2d110_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f266ae8b20d840b2b17a2b5ce1a2d110_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:09,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/7cfb9247828d403485e38a4023b9caa6, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:09,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/7cfb9247828d403485e38a4023b9caa6 is 175, key is test_row_0/A:col10/1732779668555/Put/seqid=0 2024-11-28T07:41:09,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741975_1151 (size=39549) 2024-11-28T07:41:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T07:41:09,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T07:41:09,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:09,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779729241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,246 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779729250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,255 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779729260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779729270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779729270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T07:41:09,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T07:41:09,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T07:41:09,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:09,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,492 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/7cfb9247828d403485e38a4023b9caa6 2024-11-28T07:41:09,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/9cc18df9eac6489b91a8dfd6dc21c8df is 50, key is test_row_0/B:col10/1732779668555/Put/seqid=0 2024-11-28T07:41:09,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741976_1152 (size=12001) 2024-11-28T07:41:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T07:41:09,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T07:41:09,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:09,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779729770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T07:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779729781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779729781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:09,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:09,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T07:41:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:09,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:09,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:09,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/9cc18df9eac6489b91a8dfd6dc21c8df 2024-11-28T07:41:09,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/7018cff82ba54b0b958cb57311da88a1 is 50, key is test_row_0/C:col10/1732779668555/Put/seqid=0 2024-11-28T07:41:10,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741977_1153 (size=12001) 2024-11-28T07:41:10,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/7018cff82ba54b0b958cb57311da88a1 2024-11-28T07:41:10,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/7cfb9247828d403485e38a4023b9caa6 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/7cfb9247828d403485e38a4023b9caa6 2024-11-28T07:41:10,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/7cfb9247828d403485e38a4023b9caa6, entries=200, sequenceid=63, filesize=38.6 K 2024-11-28T07:41:10,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/9cc18df9eac6489b91a8dfd6dc21c8df as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9cc18df9eac6489b91a8dfd6dc21c8df 2024-11-28T07:41:10,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9cc18df9eac6489b91a8dfd6dc21c8df, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T07:41:10,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/7018cff82ba54b0b958cb57311da88a1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7018cff82ba54b0b958cb57311da88a1 2024-11-28T07:41:10,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7018cff82ba54b0b958cb57311da88a1, entries=150, sequenceid=63, filesize=11.7 K 2024-11-28T07:41:10,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4e9a306a7a5304212409c030a7b68ce0 in 1508ms, sequenceid=63, compaction requested=true 2024-11-28T07:41:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:10,063 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:10,064 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:10,068 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:10,068 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:10,068 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:10,069 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/6c51071ef74246cc9812ed892f0ff07d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dfeace86b5d9400ab85a6b722eb3fe67, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9cc18df9eac6489b91a8dfd6dc21c8df] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=35.2 K 2024-11-28T07:41:10,070 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:10,070 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c51071ef74246cc9812ed892f0ff07d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732779666867 2024-11-28T07:41:10,070 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:10,070 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:10,070 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/555ee48a099d431c96f26ae2aff22aca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/59a3d52722014cc78cc73b2ffd543ef2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/7cfb9247828d403485e38a4023b9caa6] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=124.3 K 2024-11-28T07:41:10,070 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting dfeace86b5d9400ab85a6b722eb3fe67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732779667029 2024-11-28T07:41:10,070 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:10,071 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/555ee48a099d431c96f26ae2aff22aca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/59a3d52722014cc78cc73b2ffd543ef2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/7cfb9247828d403485e38a4023b9caa6] 2024-11-28T07:41:10,072 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cc18df9eac6489b91a8dfd6dc21c8df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732779668195 2024-11-28T07:41:10,074 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 555ee48a099d431c96f26ae2aff22aca, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732779666867 2024-11-28T07:41:10,075 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59a3d52722014cc78cc73b2ffd543ef2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732779667029 2024-11-28T07:41:10,076 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cfb9247828d403485e38a4023b9caa6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732779668195 2024-11-28T07:41:10,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-28T07:41:10,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:10,086 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T07:41:10,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:10,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:10,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:10,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:10,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:10,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:10,097 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:10,099 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:10,100 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/4e6d7ea87523481b82cbf97f1fc7ecad is 50, key is test_row_0/B:col10/1732779668555/Put/seqid=0 2024-11-28T07:41:10,103 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128a8b8a6115463420bb179afae8d4d6792_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:10,114 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128a8b8a6115463420bb179afae8d4d6792_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:10,115 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a8b8a6115463420bb179afae8d4d6792_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:10,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e185e97806924fb8a6780ca1501c840a_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779668605/Put/seqid=0 2024-11-28T07:41:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T07:41:10,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741978_1154 (size=12104) 2024-11-28T07:41:10,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741979_1155 (size=4469) 2024-11-28T07:41:10,157 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#136 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:10,158 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6c6e9f5f04684c5a8652550099b4d4e5 is 175, key is test_row_0/A:col10/1732779668555/Put/seqid=0 2024-11-28T07:41:10,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741980_1156 (size=12154) 2024-11-28T07:41:10,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741981_1157 (size=31058) 2024-11-28T07:41:10,212 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6c6e9f5f04684c5a8652550099b4d4e5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6c6e9f5f04684c5a8652550099b4d4e5 2024-11-28T07:41:10,219 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into 6c6e9f5f04684c5a8652550099b4d4e5(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:10,219 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:10,219 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=13, startTime=1732779670063; duration=0sec 2024-11-28T07:41:10,219 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:10,219 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:10,221 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:10,222 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:10,222 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:10,222 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:10,222 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/31ae9951fed4481ebf97682afd2ecabf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/3102c0aeaf734fd18c573c5a37cf899b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7018cff82ba54b0b958cb57311da88a1] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=35.2 K 2024-11-28T07:41:10,223 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31ae9951fed4481ebf97682afd2ecabf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732779666867 2024-11-28T07:41:10,223 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3102c0aeaf734fd18c573c5a37cf899b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732779667029 2024-11-28T07:41:10,224 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7018cff82ba54b0b958cb57311da88a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732779668195 2024-11-28T07:41:10,233 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:10,233 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/55592724e80447738258df4cab9cbc9c is 50, key is test_row_0/C:col10/1732779668555/Put/seqid=0 2024-11-28T07:41:10,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741982_1158 (size=12104) 2024-11-28T07:41:10,551 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/4e6d7ea87523481b82cbf97f1fc7ecad as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e6d7ea87523481b82cbf97f1fc7ecad 2024-11-28T07:41:10,563 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into 4e6d7ea87523481b82cbf97f1fc7ecad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:10,564 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:10,564 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=13, startTime=1732779670063; duration=0sec 2024-11-28T07:41:10,564 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:10,564 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:10,586 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e185e97806924fb8a6780ca1501c840a_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e185e97806924fb8a6780ca1501c840a_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:10,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/89c8c04f8edd470c8c0d44b55a8801b6, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:10,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/89c8c04f8edd470c8c0d44b55a8801b6 is 175, key is test_row_0/A:col10/1732779668605/Put/seqid=0 2024-11-28T07:41:10,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741983_1159 (size=30955) 2024-11-28T07:41:10,656 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/55592724e80447738258df4cab9cbc9c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/55592724e80447738258df4cab9cbc9c 2024-11-28T07:41:10,665 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into 55592724e80447738258df4cab9cbc9c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:10,665 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:10,665 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=13, startTime=1732779670064; duration=0sec 2024-11-28T07:41:10,666 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:10,666 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:10,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:10,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779730821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:10,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779730825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:10,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779730826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:10,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779730940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779730940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:10,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779730951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:10,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,035 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/89c8c04f8edd470c8c0d44b55a8801b6 2024-11-28T07:41:11,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2aec996170284282a1fdcf0297dd5501 is 50, key is test_row_0/B:col10/1732779668605/Put/seqid=0 2024-11-28T07:41:11,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741984_1160 (size=12001) 2024-11-28T07:41:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T07:41:11,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779731150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779731151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779731160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779731260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,266 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779731281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,284 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,456 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2aec996170284282a1fdcf0297dd5501 2024-11-28T07:41:11,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779731460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0de2f47bfba549ea8fc5f546ef0365e4 is 50, key is test_row_0/C:col10/1732779668605/Put/seqid=0 2024-11-28T07:41:11,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779731461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779731470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:11,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741985_1161 (size=12001) 2024-11-28T07:41:11,492 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0de2f47bfba549ea8fc5f546ef0365e4 2024-11-28T07:41:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/89c8c04f8edd470c8c0d44b55a8801b6 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/89c8c04f8edd470c8c0d44b55a8801b6 2024-11-28T07:41:11,511 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/89c8c04f8edd470c8c0d44b55a8801b6, entries=150, sequenceid=83, filesize=30.2 K 2024-11-28T07:41:11,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2aec996170284282a1fdcf0297dd5501 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aec996170284282a1fdcf0297dd5501 2024-11-28T07:41:11,532 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aec996170284282a1fdcf0297dd5501, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T07:41:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0de2f47bfba549ea8fc5f546ef0365e4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0de2f47bfba549ea8fc5f546ef0365e4 2024-11-28T07:41:11,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0de2f47bfba549ea8fc5f546ef0365e4, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T07:41:11,547 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4e9a306a7a5304212409c030a7b68ce0 in 1462ms, sequenceid=83, compaction requested=false 2024-11-28T07:41:11,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:11,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:11,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-28T07:41:11,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-28T07:41:11,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-28T07:41:11,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5520 sec 2024-11-28T07:41:11,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.5640 sec 2024-11-28T07:41:11,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:41:11,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:11,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:11,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:11,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:11,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:12,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ca529c6d776341d89046f7714eba99bb_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779670811/Put/seqid=0 2024-11-28T07:41:12,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779732046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779732050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779732052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741986_1162 (size=14594) 2024-11-28T07:41:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779732171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779732171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779732171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779732390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779732390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779732400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,483 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:12,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128ca529c6d776341d89046f7714eba99bb_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ca529c6d776341d89046f7714eba99bb_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:12,498 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/f8748bf6ff1b4e8ab0c93b116c15c9da, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:12,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/f8748bf6ff1b4e8ab0c93b116c15c9da is 175, key is test_row_0/A:col10/1732779670811/Put/seqid=0 2024-11-28T07:41:12,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741987_1163 (size=39549) 2024-11-28T07:41:12,522 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/f8748bf6ff1b4e8ab0c93b116c15c9da 2024-11-28T07:41:12,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/0623ba91a9b94614ab58fe4e19adf4d4 is 50, key is test_row_0/B:col10/1732779670811/Put/seqid=0 2024-11-28T07:41:12,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741988_1164 (size=12001) 2024-11-28T07:41:12,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779732700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779732700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:12,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779732721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:12,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/0623ba91a9b94614ab58fe4e19adf4d4 2024-11-28T07:41:13,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/43c33846b9274d76a902e4fd73ab2938 is 50, key is test_row_0/C:col10/1732779670811/Put/seqid=0 2024-11-28T07:41:13,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741989_1165 (size=12001) 2024-11-28T07:41:13,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/43c33846b9274d76a902e4fd73ab2938 2024-11-28T07:41:13,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/f8748bf6ff1b4e8ab0c93b116c15c9da as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/f8748bf6ff1b4e8ab0c93b116c15c9da 2024-11-28T07:41:13,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/f8748bf6ff1b4e8ab0c93b116c15c9da, entries=200, sequenceid=99, filesize=38.6 K 2024-11-28T07:41:13,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/0623ba91a9b94614ab58fe4e19adf4d4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0623ba91a9b94614ab58fe4e19adf4d4 2024-11-28T07:41:13,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0623ba91a9b94614ab58fe4e19adf4d4, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T07:41:13,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/43c33846b9274d76a902e4fd73ab2938 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/43c33846b9274d76a902e4fd73ab2938 2024-11-28T07:41:13,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/43c33846b9274d76a902e4fd73ab2938, entries=150, sequenceid=99, filesize=11.7 K 2024-11-28T07:41:13,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 4e9a306a7a5304212409c030a7b68ce0 in 1128ms, sequenceid=99, compaction requested=true 2024-11-28T07:41:13,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:13,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:13,106 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:13,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:13,106 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:13,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:13,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:13,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:13,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:13,108 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:13,108 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:13,108 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,108 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e6d7ea87523481b82cbf97f1fc7ecad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aec996170284282a1fdcf0297dd5501, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0623ba91a9b94614ab58fe4e19adf4d4] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=35.3 K 2024-11-28T07:41:13,108 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:13,109 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:13,109 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,109 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6c6e9f5f04684c5a8652550099b4d4e5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/89c8c04f8edd470c8c0d44b55a8801b6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/f8748bf6ff1b4e8ab0c93b116c15c9da] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=99.2 K 2024-11-28T07:41:13,109 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,109 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6c6e9f5f04684c5a8652550099b4d4e5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/89c8c04f8edd470c8c0d44b55a8801b6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/f8748bf6ff1b4e8ab0c93b116c15c9da] 2024-11-28T07:41:13,110 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e6d7ea87523481b82cbf97f1fc7ecad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732779668195 2024-11-28T07:41:13,110 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c6e9f5f04684c5a8652550099b4d4e5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732779668195 2024-11-28T07:41:13,110 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c8c04f8edd470c8c0d44b55a8801b6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779668605 2024-11-28T07:41:13,110 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aec996170284282a1fdcf0297dd5501, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779668605 2024-11-28T07:41:13,111 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0623ba91a9b94614ab58fe4e19adf4d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732779670811 2024-11-28T07:41:13,111 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8748bf6ff1b4e8ab0c93b116c15c9da, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732779670811 2024-11-28T07:41:13,135 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:13,136 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/dee3d365907d4b6b947275f440a3c7be is 50, key is test_row_0/B:col10/1732779670811/Put/seqid=0 2024-11-28T07:41:13,153 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:13,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-28T07:41:13,161 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:13,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-28T07:41:13,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T07:41:13,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:13,166 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:13,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:13,166 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128560974a4ef594c8fb53fc5f7025b1ee0_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:13,169 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128560974a4ef594c8fb53fc5f7025b1ee0_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:13,170 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128560974a4ef594c8fb53fc5f7025b1ee0_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:13,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741990_1166 (size=12207) 2024-11-28T07:41:13,196 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/dee3d365907d4b6b947275f440a3c7be as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dee3d365907d4b6b947275f440a3c7be 2024-11-28T07:41:13,205 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into dee3d365907d4b6b947275f440a3c7be(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:13,205 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:13,205 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=13, startTime=1732779673106; duration=0sec 2024-11-28T07:41:13,205 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:13,205 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:13,205 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:13,206 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:13,207 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:13,207 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,207 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/55592724e80447738258df4cab9cbc9c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0de2f47bfba549ea8fc5f546ef0365e4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/43c33846b9274d76a902e4fd73ab2938] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=35.3 K 2024-11-28T07:41:13,207 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 55592724e80447738258df4cab9cbc9c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732779668195 2024-11-28T07:41:13,208 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de2f47bfba549ea8fc5f546ef0365e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779668605 2024-11-28T07:41:13,208 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c33846b9274d76a902e4fd73ab2938, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732779670811 2024-11-28T07:41:13,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779672042, max timestamp=1732779673213Num uniques 859; 2024-11-28T07:41:13,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:13,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779672042, max timestamp=1732779673213Num uniques 859; 2024-11-28T07:41:13,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779672042, max timestamp=1732779673213Num uniques 859; 2024-11-28T07:41:13,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:13,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:13,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741991_1167 (size=4469) 2024-11-28T07:41:13,228 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#145 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:13,229 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6cca96400c414ba5bab4f16252f168fc is 175, key is test_row_0/A:col10/1732779670811/Put/seqid=0 2024-11-28T07:41:13,232 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#146 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:13,232 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/701e236e61ae4950853935a41f7e6639 is 50, key is test_row_0/C:col10/1732779670811/Put/seqid=0 2024-11-28T07:41:13,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-28T07:41:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:13,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:13,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741992_1168 (size=31161) 2024-11-28T07:41:13,264 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6cca96400c414ba5bab4f16252f168fc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6cca96400c414ba5bab4f16252f168fc 2024-11-28T07:41:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T07:41:13,274 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into 6cca96400c414ba5bab4f16252f168fc(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:13,275 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:13,275 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=13, startTime=1732779673105; duration=0sec 2024-11-28T07:41:13,275 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:13,275 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:13,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128983e5deaf91c4fd5889a471971ddeba1_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779672042/Put/seqid=0 2024-11-28T07:41:13,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741993_1169 (size=12207) 2024-11-28T07:41:13,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T07:41:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741994_1170 (size=12154) 2024-11-28T07:41:13,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779673246, max timestamp=1732779673338Num uniques 859; 2024-11-28T07:41:13,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:13,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:13,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779673246, max timestamp=1732779673338Num uniques 859; 2024-11-28T07:41:13,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:13,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779673246, max timestamp=1732779673342Num uniques 859; 2024-11-28T07:41:13,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:13,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128983e5deaf91c4fd5889a471971ddeba1_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128983e5deaf91c4fd5889a471971ddeba1_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:13,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e65fbda126084ace862fc6f99e39d647, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:13,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e65fbda126084ace862fc6f99e39d647 is 175, key is test_row_0/A:col10/1732779672042/Put/seqid=0 2024-11-28T07:41:13,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741995_1171 (size=30955) 2024-11-28T07:41:13,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=126, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e65fbda126084ace862fc6f99e39d647 2024-11-28T07:41:13,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/5989d8ee8caf45719df8fb8d6500a44c is 50, key is test_row_0/B:col10/1732779672042/Put/seqid=0 2024-11-28T07:41:13,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779733405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779733408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779733408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741996_1172 (size=12001) 2024-11-28T07:41:13,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T07:41:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:13,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T07:41:13,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779733520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779733520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779733520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T07:41:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,717 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/701e236e61ae4950853935a41f7e6639 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/701e236e61ae4950853935a41f7e6639 2024-11-28T07:41:13,731 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into 701e236e61ae4950853935a41f7e6639(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:13,731 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:13,731 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=13, startTime=1732779673106; duration=0sec 2024-11-28T07:41:13,732 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:13,732 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:13,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779733730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779733730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779733730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:13,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T07:41:13,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:13,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T07:41:13,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/5989d8ee8caf45719df8fb8d6500a44c 2024-11-28T07:41:13,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/e8afdfdc9dae47bea63a593fa6968f45 is 50, key is test_row_0/C:col10/1732779672042/Put/seqid=0 2024-11-28T07:41:13,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741997_1173 (size=12001) 2024-11-28T07:41:13,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/e8afdfdc9dae47bea63a593fa6968f45 2024-11-28T07:41:13,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e65fbda126084ace862fc6f99e39d647 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e65fbda126084ace862fc6f99e39d647 2024-11-28T07:41:13,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e65fbda126084ace862fc6f99e39d647, entries=150, sequenceid=126, filesize=30.2 K 2024-11-28T07:41:13,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/5989d8ee8caf45719df8fb8d6500a44c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5989d8ee8caf45719df8fb8d6500a44c 2024-11-28T07:41:13,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5989d8ee8caf45719df8fb8d6500a44c, entries=150, sequenceid=126, filesize=11.7 K 2024-11-28T07:41:13,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/e8afdfdc9dae47bea63a593fa6968f45 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e8afdfdc9dae47bea63a593fa6968f45 2024-11-28T07:41:13,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e8afdfdc9dae47bea63a593fa6968f45, entries=150, sequenceid=126, filesize=11.7 K 2024-11-28T07:41:13,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=134.18 KB/137400 for 4e9a306a7a5304212409c030a7b68ce0 in 661ms, sequenceid=126, compaction requested=false 2024-11-28T07:41:13,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:13,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-28T07:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:13,936 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T07:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:13,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112835e69d410f9e4fc990fb1f746b2325da_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779673406/Put/seqid=0 2024-11-28T07:41:13,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741998_1174 (size=12304) 2024-11-28T07:41:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:14,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:14,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779734063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779734064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779734066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779734170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779734170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779734181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T07:41:14,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:14,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779734380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,388 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112835e69d410f9e4fc990fb1f746b2325da_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112835e69d410f9e4fc990fb1f746b2325da_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:14,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779734390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e1843340041c414081071b99295ff32f, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:14,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e1843340041c414081071b99295ff32f is 175, key is test_row_0/A:col10/1732779673406/Put/seqid=0 2024-11-28T07:41:14,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779734390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741999_1175 (size=31105) 2024-11-28T07:41:14,405 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e1843340041c414081071b99295ff32f 2024-11-28T07:41:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/0895bea735204ee4805fff891339acc3 is 50, key is test_row_0/B:col10/1732779673406/Put/seqid=0 2024-11-28T07:41:14,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742000_1176 (size=12151) 2024-11-28T07:41:14,439 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/0895bea735204ee4805fff891339acc3 2024-11-28T07:41:14,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/c839667856a547d89fbbf2c96b382e87 is 50, key is test_row_0/C:col10/1732779673406/Put/seqid=0 2024-11-28T07:41:14,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742001_1177 (size=12151) 2024-11-28T07:41:14,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779734690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779734700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779734710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:14,857 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/c839667856a547d89fbbf2c96b382e87 2024-11-28T07:41:14,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/e1843340041c414081071b99295ff32f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e1843340041c414081071b99295ff32f 2024-11-28T07:41:14,867 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e1843340041c414081071b99295ff32f, entries=150, sequenceid=165, filesize=30.4 K 2024-11-28T07:41:14,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/0895bea735204ee4805fff891339acc3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0895bea735204ee4805fff891339acc3 2024-11-28T07:41:14,872 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0895bea735204ee4805fff891339acc3, entries=150, sequenceid=165, filesize=11.9 K 2024-11-28T07:41:14,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/c839667856a547d89fbbf2c96b382e87 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c839667856a547d89fbbf2c96b382e87 2024-11-28T07:41:14,878 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c839667856a547d89fbbf2c96b382e87, entries=150, sequenceid=165, filesize=11.9 K 2024-11-28T07:41:14,879 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4e9a306a7a5304212409c030a7b68ce0 in 943ms, sequenceid=165, compaction requested=true 2024-11-28T07:41:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:14,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-28T07:41:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-28T07:41:14,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-28T07:41:14,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7150 sec 2024-11-28T07:41:14,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7220 sec 2024-11-28T07:41:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:15,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:41:15,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:15,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:15,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:15,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:15,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:15,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:15,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b320e5dade3c4b2384a468400d0b3e88_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779674051/Put/seqid=0 2024-11-28T07:41:15,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742002_1178 (size=14794) 2024-11-28T07:41:15,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779735256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779675236, max timestamp=1732779675259Num uniques 859; 2024-11-28T07:41:15,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:15,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779675236, max timestamp=1732779675260Num uniques 859; 2024-11-28T07:41:15,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779675236, max timestamp=1732779675259Num uniques 859; 2024-11-28T07:41:15,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:15,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-28T07:41:15,311 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:15,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-28T07:41:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T07:41:15,314 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:15,316 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:15,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:15,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779735310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779735316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,323 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8274 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779735322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779735324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779735370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T07:41:15,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779735430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779735440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779735440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:15,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:15,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779735581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,623 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:15,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b320e5dade3c4b2384a468400d0b3e88_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b320e5dade3c4b2384a468400d0b3e88_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T07:41:15,631 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/2a1b30e85fb64a2ab98937b5032b7134, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:15,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/2a1b30e85fb64a2ab98937b5032b7134 is 175, key is test_row_0/A:col10/1732779674051/Put/seqid=0 2024-11-28T07:41:15,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742003_1179 (size=39749) 2024-11-28T07:41:15,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779735640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779735650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779735651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:15,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:15,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779735900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:15,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:15,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T07:41:15,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779735950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779735970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779735970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:16,036 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/2a1b30e85fb64a2ab98937b5032b7134 2024-11-28T07:41:16,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/62fbdeef020b4fabb102d59b13fce85b is 50, key is test_row_0/B:col10/1732779674051/Put/seqid=0 2024-11-28T07:41:16,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742004_1180 (size=12151) 2024-11-28T07:41:16,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/62fbdeef020b4fabb102d59b13fce85b 2024-11-28T07:41:16,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/52cc25d44f3c49d1821a4ad011c44112 is 50, key is test_row_0/C:col10/1732779674051/Put/seqid=0 2024-11-28T07:41:16,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742005_1181 (size=12151) 2024-11-28T07:41:16,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:16,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:16,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779736420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T07:41:16,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:16,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779736460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:16,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/52cc25d44f3c49d1821a4ad011c44112 2024-11-28T07:41:16,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/2a1b30e85fb64a2ab98937b5032b7134 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/2a1b30e85fb64a2ab98937b5032b7134 2024-11-28T07:41:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779736480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779736481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:16,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/2a1b30e85fb64a2ab98937b5032b7134, entries=200, sequenceid=176, filesize=38.8 K 2024-11-28T07:41:16,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/62fbdeef020b4fabb102d59b13fce85b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/62fbdeef020b4fabb102d59b13fce85b 2024-11-28T07:41:16,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/62fbdeef020b4fabb102d59b13fce85b, entries=150, sequenceid=176, filesize=11.9 K 2024-11-28T07:41:16,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/52cc25d44f3c49d1821a4ad011c44112 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/52cc25d44f3c49d1821a4ad011c44112 2024-11-28T07:41:16,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/52cc25d44f3c49d1821a4ad011c44112, entries=150, sequenceid=176, filesize=11.9 K 2024-11-28T07:41:16,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4e9a306a7a5304212409c030a7b68ce0 in 1295ms, sequenceid=176, compaction requested=true 2024-11-28T07:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:16,497 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:16,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T07:41:16,497 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:41:16,500 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:41:16,500 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:16,500 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,500 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6cca96400c414ba5bab4f16252f168fc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e65fbda126084ace862fc6f99e39d647, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e1843340041c414081071b99295ff32f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/2a1b30e85fb64a2ab98937b5032b7134] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=129.9 K 2024-11-28T07:41:16,500 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,500 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6cca96400c414ba5bab4f16252f168fc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e65fbda126084ace862fc6f99e39d647, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e1843340041c414081071b99295ff32f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/2a1b30e85fb64a2ab98937b5032b7134] 2024-11-28T07:41:16,501 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:41:16,501 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cca96400c414ba5bab4f16252f168fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732779670811 2024-11-28T07:41:16,501 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:16,501 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,501 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dee3d365907d4b6b947275f440a3c7be, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5989d8ee8caf45719df8fb8d6500a44c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0895bea735204ee4805fff891339acc3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/62fbdeef020b4fabb102d59b13fce85b] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=47.4 K 2024-11-28T07:41:16,501 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65fbda126084ace862fc6f99e39d647, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732779672042 2024-11-28T07:41:16,502 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting dee3d365907d4b6b947275f440a3c7be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732779670811 2024-11-28T07:41:16,502 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1843340041c414081071b99295ff32f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732779673393 2024-11-28T07:41:16,502 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a1b30e85fb64a2ab98937b5032b7134, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779674051 2024-11-28T07:41:16,502 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5989d8ee8caf45719df8fb8d6500a44c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732779672042 2024-11-28T07:41:16,503 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0895bea735204ee4805fff891339acc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732779673393 2024-11-28T07:41:16,503 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fbdeef020b4fabb102d59b13fce85b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779674051 2024-11-28T07:41:16,521 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:16,532 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#157 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:16,533 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/fe26e3f340c440b792443fc18d346267 is 50, key is test_row_0/B:col10/1732779674051/Put/seqid=0 2024-11-28T07:41:16,534 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411287bd7a01b794c4393b4acd3b076971505_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:16,537 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411287bd7a01b794c4393b4acd3b076971505_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:16,538 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287bd7a01b794c4393b4acd3b076971505_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:16,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-28T07:41:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,546 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T07:41:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:16,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:16,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:16,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:16,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:16,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742006_1182 (size=4469) 2024-11-28T07:41:16,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742007_1183 (size=12493) 2024-11-28T07:41:16,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280d081c863e2344a9bca262b6e4ccd4db_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779675295/Put/seqid=0 2024-11-28T07:41:16,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742008_1184 (size=12304) 2024-11-28T07:41:16,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:16,627 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280d081c863e2344a9bca262b6e4ccd4db_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280d081c863e2344a9bca262b6e4ccd4db_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:16,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/c3d330a74548455f9c58633247b9c26e, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:16,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/c3d330a74548455f9c58633247b9c26e is 175, key is test_row_0/A:col10/1732779675295/Put/seqid=0 2024-11-28T07:41:16,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742009_1185 (size=31105) 2024-11-28T07:41:16,949 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:16,950 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/0d8b725204df44b88abfe6d08d9035fb is 175, key is test_row_0/A:col10/1732779674051/Put/seqid=0 2024-11-28T07:41:16,980 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/fe26e3f340c440b792443fc18d346267 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/fe26e3f340c440b792443fc18d346267 2024-11-28T07:41:16,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742010_1186 (size=31447) 2024-11-28T07:41:16,992 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/0d8b725204df44b88abfe6d08d9035fb as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/0d8b725204df44b88abfe6d08d9035fb 2024-11-28T07:41:16,992 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into fe26e3f340c440b792443fc18d346267(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:16,992 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:16,993 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=12, startTime=1732779676497; duration=0sec 2024-11-28T07:41:16,993 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:16,993 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:16,993 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:41:16,994 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:41:16,995 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:16,995 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:16,995 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/701e236e61ae4950853935a41f7e6639, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e8afdfdc9dae47bea63a593fa6968f45, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c839667856a547d89fbbf2c96b382e87, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/52cc25d44f3c49d1821a4ad011c44112] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=47.4 K 2024-11-28T07:41:16,995 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 701e236e61ae4950853935a41f7e6639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732779670811 2024-11-28T07:41:16,996 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e8afdfdc9dae47bea63a593fa6968f45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732779672042 2024-11-28T07:41:16,997 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting c839667856a547d89fbbf2c96b382e87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732779673393 2024-11-28T07:41:16,998 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 52cc25d44f3c49d1821a4ad011c44112, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779674051 2024-11-28T07:41:16,999 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into 0d8b725204df44b88abfe6d08d9035fb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:16,999 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:16,999 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=12, startTime=1732779676497; duration=0sec 2024-11-28T07:41:16,999 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:16,999 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:17,024 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:17,025 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/ef4b8c45711a45dcb1e11df2a362b421 is 50, key is test_row_0/C:col10/1732779674051/Put/seqid=0 2024-11-28T07:41:17,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742011_1187 (size=12493) 2024-11-28T07:41:17,070 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/ef4b8c45711a45dcb1e11df2a362b421 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/ef4b8c45711a45dcb1e11df2a362b421 2024-11-28T07:41:17,077 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/c3d330a74548455f9c58633247b9c26e 2024-11-28T07:41:17,087 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into ef4b8c45711a45dcb1e11df2a362b421(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:17,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:17,087 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=12, startTime=1732779676497; duration=0sec 2024-11-28T07:41:17,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:17,088 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:17,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/5dc98729a38f41ef9113eb80893ea9c2 is 50, key is test_row_0/B:col10/1732779675295/Put/seqid=0 2024-11-28T07:41:17,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742012_1188 (size=12151) 2024-11-28T07:41:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:17,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T07:41:17,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779737491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779737491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,495 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:17,495 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779737492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779737494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,502 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/5dc98729a38f41ef9113eb80893ea9c2 2024-11-28T07:41:17,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/a874c88352bc4c27951b2f64bd32dc25 is 50, key is test_row_0/C:col10/1732779675295/Put/seqid=0 2024-11-28T07:41:17,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742013_1189 (size=12151) 2024-11-28T07:41:17,555 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/a874c88352bc4c27951b2f64bd32dc25 2024-11-28T07:41:17,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/c3d330a74548455f9c58633247b9c26e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/c3d330a74548455f9c58633247b9c26e 2024-11-28T07:41:17,576 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/c3d330a74548455f9c58633247b9c26e, entries=150, sequenceid=212, filesize=30.4 K 2024-11-28T07:41:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/5dc98729a38f41ef9113eb80893ea9c2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5dc98729a38f41ef9113eb80893ea9c2 2024-11-28T07:41:17,589 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5dc98729a38f41ef9113eb80893ea9c2, entries=150, sequenceid=212, filesize=11.9 K 2024-11-28T07:41:17,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/a874c88352bc4c27951b2f64bd32dc25 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/a874c88352bc4c27951b2f64bd32dc25 2024-11-28T07:41:17,596 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/a874c88352bc4c27951b2f64bd32dc25, entries=150, sequenceid=212, filesize=11.9 K 2024-11-28T07:41:17,597 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4e9a306a7a5304212409c030a7b68ce0 in 1051ms, sequenceid=212, compaction requested=false 2024-11-28T07:41:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-28T07:41:17,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-28T07:41:17,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-28T07:41:17,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2830 sec 2024-11-28T07:41:17,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.2930 sec 2024-11-28T07:41:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:17,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:41:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:17,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112873ef4efda2d24493b54180222a4583c6_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779677614/Put/seqid=0 2024-11-28T07:41:17,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742014_1190 (size=12304) 2024-11-28T07:41:17,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:17,692 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112873ef4efda2d24493b54180222a4583c6_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112873ef4efda2d24493b54180222a4583c6_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:17,695 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/d1aa02abfffd44208bd8aebf4a0825d2, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:17,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/d1aa02abfffd44208bd8aebf4a0825d2 is 175, key is test_row_0/A:col10/1732779677614/Put/seqid=0 2024-11-28T07:41:17,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779737711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779677657, max timestamp=1732779677712Num uniques 859; 2024-11-28T07:41:17,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:17,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779677657, max timestamp=1732779677712Num uniques 859; 2024-11-28T07:41:17,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779677657, max timestamp=1732779677712Num uniques 859; 2024-11-28T07:41:17,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:17,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742015_1191 (size=31105) 2024-11-28T07:41:17,735 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/d1aa02abfffd44208bd8aebf4a0825d2 2024-11-28T07:41:17,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/aeba99ebd03049ba81fbc3187c47a0d3 is 50, key is test_row_0/B:col10/1732779677614/Put/seqid=0 2024-11-28T07:41:17,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742016_1192 (size=12151) 2024-11-28T07:41:17,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779737851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779737852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779737970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:17,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779737970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779738180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779738180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/aeba99ebd03049ba81fbc3187c47a0d3 2024-11-28T07:41:18,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/9a6837705ac946959b22174f89867323 is 50, key is test_row_0/C:col10/1732779677614/Put/seqid=0 2024-11-28T07:41:18,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742017_1193 (size=12151) 2024-11-28T07:41:18,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/9a6837705ac946959b22174f89867323 2024-11-28T07:41:18,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/d1aa02abfffd44208bd8aebf4a0825d2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d1aa02abfffd44208bd8aebf4a0825d2 2024-11-28T07:41:18,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d1aa02abfffd44208bd8aebf4a0825d2, entries=150, sequenceid=226, filesize=30.4 K 2024-11-28T07:41:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/aeba99ebd03049ba81fbc3187c47a0d3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/aeba99ebd03049ba81fbc3187c47a0d3 2024-11-28T07:41:18,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/aeba99ebd03049ba81fbc3187c47a0d3, entries=150, sequenceid=226, filesize=11.9 K 2024-11-28T07:41:18,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/9a6837705ac946959b22174f89867323 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/9a6837705ac946959b22174f89867323 2024-11-28T07:41:18,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/9a6837705ac946959b22174f89867323, entries=150, sequenceid=226, filesize=11.9 K 2024-11-28T07:41:18,267 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-28T07:41:18,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4e9a306a7a5304212409c030a7b68ce0 in 653ms, sequenceid=226, compaction requested=true 2024-11-28T07:41:18,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:18,269 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:18,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:18,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:18,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:18,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:18,270 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:18,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:18,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:18,271 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:18,271 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:18,271 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:18,271 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/0d8b725204df44b88abfe6d08d9035fb, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/c3d330a74548455f9c58633247b9c26e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d1aa02abfffd44208bd8aebf4a0825d2] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=91.5 K 2024-11-28T07:41:18,271 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:18,272 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/0d8b725204df44b88abfe6d08d9035fb, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/c3d330a74548455f9c58633247b9c26e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d1aa02abfffd44208bd8aebf4a0825d2] 2024-11-28T07:41:18,272 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:18,272 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:18,272 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:18,272 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/fe26e3f340c440b792443fc18d346267, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5dc98729a38f41ef9113eb80893ea9c2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/aeba99ebd03049ba81fbc3187c47a0d3] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=35.9 K 2024-11-28T07:41:18,273 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d8b725204df44b88abfe6d08d9035fb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779674051 2024-11-28T07:41:18,273 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting fe26e3f340c440b792443fc18d346267, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779674051 2024-11-28T07:41:18,273 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3d330a74548455f9c58633247b9c26e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732779675295 2024-11-28T07:41:18,273 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dc98729a38f41ef9113eb80893ea9c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732779675295 2024-11-28T07:41:18,274 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1aa02abfffd44208bd8aebf4a0825d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779677442 2024-11-28T07:41:18,274 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting aeba99ebd03049ba81fbc3187c47a0d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779677442 2024-11-28T07:41:18,299 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:18,300 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2aa4efde72a442c49754186b2d3e8ce5 is 50, key is test_row_0/B:col10/1732779677614/Put/seqid=0 2024-11-28T07:41:18,302 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:18,329 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128c8d50b6834334803866d494cf6e2f0b1_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:18,331 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128c8d50b6834334803866d494cf6e2f0b1_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:18,332 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c8d50b6834334803866d494cf6e2f0b1_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:18,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742019_1195 (size=4469) 2024-11-28T07:41:18,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742018_1194 (size=12595) 2024-11-28T07:41:18,366 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2aa4efde72a442c49754186b2d3e8ce5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aa4efde72a442c49754186b2d3e8ce5 2024-11-28T07:41:18,372 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into 2aa4efde72a442c49754186b2d3e8ce5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:18,372 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:18,373 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=13, startTime=1732779678269; duration=0sec 2024-11-28T07:41:18,373 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:18,373 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:18,373 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:18,374 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:18,374 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:18,374 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:18,374 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/ef4b8c45711a45dcb1e11df2a362b421, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/a874c88352bc4c27951b2f64bd32dc25, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/9a6837705ac946959b22174f89867323] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=35.9 K 2024-11-28T07:41:18,375 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ef4b8c45711a45dcb1e11df2a362b421, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779674051 2024-11-28T07:41:18,375 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a874c88352bc4c27951b2f64bd32dc25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732779675295 2024-11-28T07:41:18,375 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6837705ac946959b22174f89867323, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779677442 2024-11-28T07:41:18,382 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:18,382 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f42ce6d6378346328a315b07d8386cd5 is 50, key is test_row_0/C:col10/1732779677614/Put/seqid=0 2024-11-28T07:41:18,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742020_1196 (size=12595) 2024-11-28T07:41:18,416 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f42ce6d6378346328a315b07d8386cd5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f42ce6d6378346328a315b07d8386cd5 2024-11-28T07:41:18,433 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into f42ce6d6378346328a315b07d8386cd5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:18,433 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:18,433 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=13, startTime=1732779678270; duration=0sec 2024-11-28T07:41:18,433 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:18,433 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:18,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T07:41:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:18,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:18,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112883a8f9a8f50c4bec8cd5544323394a13_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779678494/Put/seqid=0 2024-11-28T07:41:18,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779738517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779738519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742021_1197 (size=17534) 2024-11-28T07:41:18,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779738630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779738630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,758 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#166 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:18,759 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/d9a6a120169b4efa82e4f66d060ef85e is 175, key is test_row_0/A:col10/1732779677614/Put/seqid=0 2024-11-28T07:41:18,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742022_1198 (size=31549) 2024-11-28T07:41:18,773 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/d9a6a120169b4efa82e4f66d060ef85e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d9a6a120169b4efa82e4f66d060ef85e 2024-11-28T07:41:18,780 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into d9a6a120169b4efa82e4f66d060ef85e(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:18,780 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:18,780 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=13, startTime=1732779678269; duration=0sec 2024-11-28T07:41:18,780 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:18,780 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779738840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779738840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:18,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:18,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112883a8f9a8f50c4bec8cd5544323394a13_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883a8f9a8f50c4bec8cd5544323394a13_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:18,965 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/a6e1b68fce0b44bcaf1220e1650626d0, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:18,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/a6e1b68fce0b44bcaf1220e1650626d0 is 175, key is test_row_0/A:col10/1732779678494/Put/seqid=0 2024-11-28T07:41:18,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742023_1199 (size=48639) 2024-11-28T07:41:18,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/a6e1b68fce0b44bcaf1220e1650626d0 2024-11-28T07:41:19,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/1c8bc51760d740e69df8cc9dc2e479d8 is 50, key is test_row_0/B:col10/1732779678494/Put/seqid=0 2024-11-28T07:41:19,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742024_1200 (size=12301) 2024-11-28T07:41:19,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779739150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779739150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/1c8bc51760d740e69df8cc9dc2e479d8 2024-11-28T07:41:19,448 INFO [master/f64cf8bdf19b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T07:41:19,448 INFO [master/f64cf8bdf19b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T07:41:19,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/d563c359092748f4ba9481203492a91e is 50, key is test_row_0/C:col10/1732779678494/Put/seqid=0 2024-11-28T07:41:19,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742025_1201 (size=12301) 2024-11-28T07:41:19,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/d563c359092748f4ba9481203492a91e 2024-11-28T07:41:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-28T07:41:19,481 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:19,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-28T07:41:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T07:41:19,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/a6e1b68fce0b44bcaf1220e1650626d0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/a6e1b68fce0b44bcaf1220e1650626d0 2024-11-28T07:41:19,484 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:19,485 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:19,485 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:19,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/a6e1b68fce0b44bcaf1220e1650626d0, entries=250, sequenceid=265, filesize=47.5 K 2024-11-28T07:41:19,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/1c8bc51760d740e69df8cc9dc2e479d8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/1c8bc51760d740e69df8cc9dc2e479d8 2024-11-28T07:41:19,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/1c8bc51760d740e69df8cc9dc2e479d8, entries=150, sequenceid=265, filesize=12.0 K 2024-11-28T07:41:19,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/d563c359092748f4ba9481203492a91e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/d563c359092748f4ba9481203492a91e 2024-11-28T07:41:19,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/d563c359092748f4ba9481203492a91e, entries=150, sequenceid=265, filesize=12.0 K 2024-11-28T07:41:19,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 4e9a306a7a5304212409c030a7b68ce0 in 1007ms, sequenceid=265, compaction requested=false 2024-11-28T07:41:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:19,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:41:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:19,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b78a6884b72945719cd2214eba0c14ad_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779679515/Put/seqid=0 2024-11-28T07:41:19,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742026_1202 (size=17534) 2024-11-28T07:41:19,565 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:19,574 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b78a6884b72945719cd2214eba0c14ad_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b78a6884b72945719cd2214eba0c14ad_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:19,578 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4a34c83fb0564d5e904a6554ff13bf1e, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:19,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4a34c83fb0564d5e904a6554ff13bf1e is 175, key is test_row_0/A:col10/1732779679515/Put/seqid=0 2024-11-28T07:41:19,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T07:41:19,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779679568, max timestamp=1732779679598Num uniques 859; 2024-11-28T07:41:19,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779679568, max timestamp=1732779679598Num uniques 859; 2024-11-28T07:41:19,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:41:19,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:41:19,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779679576, max timestamp=1732779679598Num uniques 859; 2024-11-28T07:41:19,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:19,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742027_1203 (size=48639) 2024-11-28T07:41:19,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4a34c83fb0564d5e904a6554ff13bf1e 2024-11-28T07:41:19,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/e65a4aa51d5f4642aaede468614836a1 is 50, key is test_row_0/B:col10/1732779679515/Put/seqid=0 2024-11-28T07:41:19,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T07:41:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:19,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742028_1204 (size=12301) 2024-11-28T07:41:19,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779739674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779739676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779739676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779739687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779739791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779739791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T07:41:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:19,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779739791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T07:41:19,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779739800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:19,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:19,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T07:41:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:19,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:19,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779740004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779740005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779740006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779740010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/e65a4aa51d5f4642aaede468614836a1 2024-11-28T07:41:20,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/b00420fe26534cada8decb3b16fe4b6a is 50, key is test_row_0/C:col10/1732779679515/Put/seqid=0 2024-11-28T07:41:20,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T07:41:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T07:41:20,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742029_1205 (size=12301) 2024-11-28T07:41:20,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T07:41:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779740310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779740320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779740321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779740321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T07:41:20,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:20,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:20,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/b00420fe26534cada8decb3b16fe4b6a 2024-11-28T07:41:20,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4a34c83fb0564d5e904a6554ff13bf1e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4a34c83fb0564d5e904a6554ff13bf1e 2024-11-28T07:41:20,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4a34c83fb0564d5e904a6554ff13bf1e, entries=250, sequenceid=277, filesize=47.5 K 2024-11-28T07:41:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/e65a4aa51d5f4642aaede468614836a1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e65a4aa51d5f4642aaede468614836a1 2024-11-28T07:41:20,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e65a4aa51d5f4642aaede468614836a1, entries=150, sequenceid=277, filesize=12.0 K 2024-11-28T07:41:20,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/b00420fe26534cada8decb3b16fe4b6a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/b00420fe26534cada8decb3b16fe4b6a 2024-11-28T07:41:20,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/b00420fe26534cada8decb3b16fe4b6a, entries=150, sequenceid=277, filesize=12.0 K 2024-11-28T07:41:20,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 4e9a306a7a5304212409c030a7b68ce0 in 1040ms, sequenceid=277, compaction requested=true 2024-11-28T07:41:20,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:20,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:20,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:20,558 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:20,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:20,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:20,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:20,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:20,558 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:20,559 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:20,559 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:20,559 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,559 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d9a6a120169b4efa82e4f66d060ef85e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/a6e1b68fce0b44bcaf1220e1650626d0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4a34c83fb0564d5e904a6554ff13bf1e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=125.8 K 2024-11-28T07:41:20,559 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,559 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d9a6a120169b4efa82e4f66d060ef85e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/a6e1b68fce0b44bcaf1220e1650626d0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4a34c83fb0564d5e904a6554ff13bf1e] 2024-11-28T07:41:20,560 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:20,560 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:20,560 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,560 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aa4efde72a442c49754186b2d3e8ce5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/1c8bc51760d740e69df8cc9dc2e479d8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e65a4aa51d5f4642aaede468614836a1] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=36.3 K 2024-11-28T07:41:20,560 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9a6a120169b4efa82e4f66d060ef85e, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779677442 2024-11-28T07:41:20,560 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6e1b68fce0b44bcaf1220e1650626d0, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732779677835 2024-11-28T07:41:20,560 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa4efde72a442c49754186b2d3e8ce5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779677442 2024-11-28T07:41:20,562 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8bc51760d740e69df8cc9dc2e479d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732779677850 2024-11-28T07:41:20,562 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a34c83fb0564d5e904a6554ff13bf1e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732779678503 2024-11-28T07:41:20,562 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e65a4aa51d5f4642aaede468614836a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732779678503 2024-11-28T07:41:20,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-28T07:41:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,571 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-28T07:41:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:20,577 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:20,583 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:20,584 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/ab72d7646cd64b28bb1d94547088d549 is 50, key is test_row_0/B:col10/1732779679515/Put/seqid=0 2024-11-28T07:41:20,585 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411286edef2aa64f545fb838b77af2f6dff3f_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:20,587 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411286edef2aa64f545fb838b77af2f6dff3f_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:20,587 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411286edef2aa64f545fb838b77af2f6dff3f_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:20,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e67636ed5aec43c9b9b14093a02a4a2c_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779679672/Put/seqid=0 2024-11-28T07:41:20,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742030_1206 (size=12847) 2024-11-28T07:41:20,612 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/ab72d7646cd64b28bb1d94547088d549 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/ab72d7646cd64b28bb1d94547088d549 2024-11-28T07:41:20,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742031_1207 (size=4469) 2024-11-28T07:41:20,620 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#174 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T07:41:20,621 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/76d71b2a5846452d89abd8fd9cf6e42b is 175, key is test_row_0/A:col10/1732779679515/Put/seqid=0 2024-11-28T07:41:20,621 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into ab72d7646cd64b28bb1d94547088d549(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:20,621 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:20,623 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=13, startTime=1732779680558; duration=0sec 2024-11-28T07:41:20,623 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:20,623 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:20,623 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:20,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742032_1208 (size=12454) 2024-11-28T07:41:20,624 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:20,624 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:20,625 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:20,625 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f42ce6d6378346328a315b07d8386cd5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/d563c359092748f4ba9481203492a91e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/b00420fe26534cada8decb3b16fe4b6a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=36.3 K 2024-11-28T07:41:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:20,626 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f42ce6d6378346328a315b07d8386cd5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779677442 2024-11-28T07:41:20,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742033_1209 (size=31801) 2024-11-28T07:41:20,628 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d563c359092748f4ba9481203492a91e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732779677850 2024-11-28T07:41:20,629 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting b00420fe26534cada8decb3b16fe4b6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732779678503 2024-11-28T07:41:20,634 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e67636ed5aec43c9b9b14093a02a4a2c_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e67636ed5aec43c9b9b14093a02a4a2c_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:20,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8c522228bbbd4ce3ac0e511d5c3a676d, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:20,636 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/76d71b2a5846452d89abd8fd9cf6e42b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/76d71b2a5846452d89abd8fd9cf6e42b 2024-11-28T07:41:20,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8c522228bbbd4ce3ac0e511d5c3a676d is 175, key is test_row_0/A:col10/1732779679672/Put/seqid=0 2024-11-28T07:41:20,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742034_1210 (size=31255) 2024-11-28T07:41:20,646 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:20,646 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0f6893cc3fd544a9ae82aded3686ef36 is 50, key is test_row_0/C:col10/1732779679515/Put/seqid=0 2024-11-28T07:41:20,647 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into 76d71b2a5846452d89abd8fd9cf6e42b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:20,647 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:20,647 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=13, startTime=1732779680558; duration=0sec 2024-11-28T07:41:20,647 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:20,647 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:20,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742035_1211 (size=12847) 2024-11-28T07:41:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:20,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:20,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779740838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779740838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779740839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779740840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779740950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779740950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779740951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779740951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,046 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8c522228bbbd4ce3ac0e511d5c3a676d 2024-11-28T07:41:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/e93cbeed0e974f7581f0d36e23e4cb8b is 50, key is test_row_0/B:col10/1732779679672/Put/seqid=0 2024-11-28T07:41:21,067 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0f6893cc3fd544a9ae82aded3686ef36 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0f6893cc3fd544a9ae82aded3686ef36 2024-11-28T07:41:21,075 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into 0f6893cc3fd544a9ae82aded3686ef36(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:21,075 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:21,075 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=13, startTime=1732779680558; duration=0sec 2024-11-28T07:41:21,075 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:21,075 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:21,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742036_1212 (size=12301) 2024-11-28T07:41:21,085 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/e93cbeed0e974f7581f0d36e23e4cb8b 2024-11-28T07:41:21,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f8fad082757146f1b13a46f81f5edef9 is 50, key is test_row_0/C:col10/1732779679672/Put/seqid=0 2024-11-28T07:41:21,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742037_1213 (size=12301) 2024-11-28T07:41:21,137 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f8fad082757146f1b13a46f81f5edef9 2024-11-28T07:41:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8c522228bbbd4ce3ac0e511d5c3a676d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8c522228bbbd4ce3ac0e511d5c3a676d 2024-11-28T07:41:21,153 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8c522228bbbd4ce3ac0e511d5c3a676d, entries=150, sequenceid=312, filesize=30.5 K 2024-11-28T07:41:21,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/e93cbeed0e974f7581f0d36e23e4cb8b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e93cbeed0e974f7581f0d36e23e4cb8b 2024-11-28T07:41:21,162 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e93cbeed0e974f7581f0d36e23e4cb8b, entries=150, sequenceid=312, filesize=12.0 K 2024-11-28T07:41:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f8fad082757146f1b13a46f81f5edef9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f8fad082757146f1b13a46f81f5edef9 2024-11-28T07:41:21,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779741160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779741160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,169 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f8fad082757146f1b13a46f81f5edef9, entries=150, sequenceid=312, filesize=12.0 K 2024-11-28T07:41:21,170 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=53.67 KB/54960 for 4e9a306a7a5304212409c030a7b68ce0 in 599ms, sequenceid=312, compaction requested=false 2024-11-28T07:41:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-28T07:41:21,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-28T07:41:21,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-28T07:41:21,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6870 sec 2024-11-28T07:41:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:21,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:41:21,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6950 sec 2024-11-28T07:41:21,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:21,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:21,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:21,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:21,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:21,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:21,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112896e6ce2db0f24be588346c88746644a6_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779680837/Put/seqid=0 2024-11-28T07:41:21,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742038_1214 (size=14994) 2024-11-28T07:41:21,257 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:21,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779741253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,262 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112896e6ce2db0f24be588346c88746644a6_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112896e6ce2db0f24be588346c88746644a6_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:21,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779741253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,263 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4ad9b142b930428dab96936b07763d12, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:21,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4ad9b142b930428dab96936b07763d12 is 175, key is test_row_0/A:col10/1732779680837/Put/seqid=0 2024-11-28T07:41:21,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742039_1215 (size=39949) 2024-11-28T07:41:21,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779741371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779741371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779741471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779741480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779741581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779741581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-28T07:41:21,631 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:21,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-28T07:41:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T07:41:21,634 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:21,635 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:21,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:21,692 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4ad9b142b930428dab96936b07763d12 2024-11-28T07:41:21,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/4e37f5b3cdb247a397e790a9614e971c is 50, key is test_row_0/B:col10/1732779680837/Put/seqid=0 2024-11-28T07:41:21,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742040_1216 (size=12301) 2024-11-28T07:41:21,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/4e37f5b3cdb247a397e790a9614e971c 2024-11-28T07:41:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T07:41:21,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/373ca903bc10442f996d1c03a57146e4 is 50, key is test_row_0/C:col10/1732779680837/Put/seqid=0 2024-11-28T07:41:21,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742041_1217 (size=12301) 2024-11-28T07:41:21,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T07:41:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:21,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:21,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:21,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779741900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779741901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T07:41:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:21,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:21,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T07:41:21,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779741980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:21,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779741990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:21,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:22,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T07:41:22,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:22,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:22,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:22,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/373ca903bc10442f996d1c03a57146e4 2024-11-28T07:41:22,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/4ad9b142b930428dab96936b07763d12 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4ad9b142b930428dab96936b07763d12 2024-11-28T07:41:22,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4ad9b142b930428dab96936b07763d12, entries=200, sequenceid=327, filesize=39.0 K 2024-11-28T07:41:22,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/4e37f5b3cdb247a397e790a9614e971c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e37f5b3cdb247a397e790a9614e971c 2024-11-28T07:41:22,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e37f5b3cdb247a397e790a9614e971c, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T07:41:22,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/373ca903bc10442f996d1c03a57146e4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/373ca903bc10442f996d1c03a57146e4 2024-11-28T07:41:22,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/373ca903bc10442f996d1c03a57146e4, entries=150, sequenceid=327, filesize=12.0 K 2024-11-28T07:41:22,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4e9a306a7a5304212409c030a7b68ce0 in 1020ms, sequenceid=327, compaction requested=true 2024-11-28T07:41:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:22,200 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:22,200 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:22,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:22,201 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:22,201 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:22,201 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:22,201 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:22,201 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,201 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,201 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/ab72d7646cd64b28bb1d94547088d549, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e93cbeed0e974f7581f0d36e23e4cb8b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e37f5b3cdb247a397e790a9614e971c] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=36.6 K 2024-11-28T07:41:22,202 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/76d71b2a5846452d89abd8fd9cf6e42b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8c522228bbbd4ce3ac0e511d5c3a676d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4ad9b142b930428dab96936b07763d12] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=100.6 K 2024-11-28T07:41:22,202 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,202 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/76d71b2a5846452d89abd8fd9cf6e42b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8c522228bbbd4ce3ac0e511d5c3a676d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4ad9b142b930428dab96936b07763d12] 2024-11-28T07:41:22,202 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ab72d7646cd64b28bb1d94547088d549, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732779678503 2024-11-28T07:41:22,202 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76d71b2a5846452d89abd8fd9cf6e42b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732779678503 2024-11-28T07:41:22,202 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e93cbeed0e974f7581f0d36e23e4cb8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732779679672 2024-11-28T07:41:22,203 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c522228bbbd4ce3ac0e511d5c3a676d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732779679672 2024-11-28T07:41:22,203 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e37f5b3cdb247a397e790a9614e971c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732779680837 2024-11-28T07:41:22,204 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad9b142b930428dab96936b07763d12, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732779680835 2024-11-28T07:41:22,226 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:22,227 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:22,228 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/4804107c944f4e889d7c08b9b8047e23 is 50, key is test_row_0/B:col10/1732779680837/Put/seqid=0 2024-11-28T07:41:22,229 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112855162d1c00884c098e3a2365c278c171_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:22,232 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112855162d1c00884c098e3a2365c278c171_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:22,232 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112855162d1c00884c098e3a2365c278c171_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:22,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-28T07:41:22,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,252 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T07:41:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T07:41:22,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dac6f7a14e80464aa66d4d181eb67c42_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779681244/Put/seqid=0 2024-11-28T07:41:22,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742043_1219 (size=4469) 2024-11-28T07:41:22,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742042_1218 (size=12949) 2024-11-28T07:41:22,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742044_1220 (size=12454) 2024-11-28T07:41:22,285 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/4804107c944f4e889d7c08b9b8047e23 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4804107c944f4e889d7c08b9b8047e23 2024-11-28T07:41:22,292 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into 4804107c944f4e889d7c08b9b8047e23(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:22,292 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:22,292 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=13, startTime=1732779682200; duration=0sec 2024-11-28T07:41:22,292 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:22,292 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:22,292 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:22,294 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:22,294 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:22,294 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,294 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0f6893cc3fd544a9ae82aded3686ef36, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f8fad082757146f1b13a46f81f5edef9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/373ca903bc10442f996d1c03a57146e4] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=36.6 K 2024-11-28T07:41:22,294 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f6893cc3fd544a9ae82aded3686ef36, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732779678503 2024-11-28T07:41:22,295 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f8fad082757146f1b13a46f81f5edef9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732779679672 2024-11-28T07:41:22,295 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 373ca903bc10442f996d1c03a57146e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732779680837 2024-11-28T07:41:22,311 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:22,312 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/464f360c148249c3a971623628b21903 is 50, key is test_row_0/C:col10/1732779680837/Put/seqid=0 2024-11-28T07:41:22,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742045_1221 (size=12949) 2024-11-28T07:41:22,336 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/464f360c148249c3a971623628b21903 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/464f360c148249c3a971623628b21903 2024-11-28T07:41:22,344 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into 464f360c148249c3a971623628b21903(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:22,344 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:22,344 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=13, startTime=1732779682200; duration=0sec 2024-11-28T07:41:22,344 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:22,344 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779742441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779742442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779742550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779742550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:22,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:22,675 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#184 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:22,676 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d is 175, key is test_row_0/A:col10/1732779680837/Put/seqid=0 2024-11-28T07:41:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:22,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742046_1222 (size=31903) 2024-11-28T07:41:22,682 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128dac6f7a14e80464aa66d4d181eb67c42_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dac6f7a14e80464aa66d4d181eb67c42_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:22,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/47767987edd64319bdaba00f934cef65, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/47767987edd64319bdaba00f934cef65 is 175, key is test_row_0/A:col10/1732779681244/Put/seqid=0 2024-11-28T07:41:22,693 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d 2024-11-28T07:41:22,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742047_1223 (size=31255) 2024-11-28T07:41:22,696 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/47767987edd64319bdaba00f934cef65 2024-11-28T07:41:22,700 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into b5aa4b11a8db46b0a5aefa2fe8aaf95d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:22,700 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:22,700 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=13, startTime=1732779682200; duration=0sec 2024-11-28T07:41:22,700 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:22,700 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:22,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/af12ad47bec34ec0aa340f9580d967c5 is 50, key is test_row_0/B:col10/1732779681244/Put/seqid=0 2024-11-28T07:41:22,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742048_1224 (size=12301) 2024-11-28T07:41:22,725 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/af12ad47bec34ec0aa340f9580d967c5 2024-11-28T07:41:22,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0cb01ff7e54a422da23ca2c5ca85bb3c is 50, key is test_row_0/C:col10/1732779681244/Put/seqid=0 2024-11-28T07:41:22,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742049_1225 (size=12301) 2024-11-28T07:41:22,757 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0cb01ff7e54a422da23ca2c5ca85bb3c 2024-11-28T07:41:22,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779742761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:22,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779742761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/47767987edd64319bdaba00f934cef65 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/47767987edd64319bdaba00f934cef65 2024-11-28T07:41:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T07:41:22,772 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/47767987edd64319bdaba00f934cef65, entries=150, sequenceid=347, filesize=30.5 K 2024-11-28T07:41:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/af12ad47bec34ec0aa340f9580d967c5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af12ad47bec34ec0aa340f9580d967c5 2024-11-28T07:41:22,778 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af12ad47bec34ec0aa340f9580d967c5, entries=150, sequenceid=347, filesize=12.0 K 2024-11-28T07:41:22,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/0cb01ff7e54a422da23ca2c5ca85bb3c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0cb01ff7e54a422da23ca2c5ca85bb3c 2024-11-28T07:41:22,785 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0cb01ff7e54a422da23ca2c5ca85bb3c, entries=150, sequenceid=347, filesize=12.0 K 2024-11-28T07:41:22,790 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4e9a306a7a5304212409c030a7b68ce0 in 538ms, sequenceid=347, compaction requested=false 2024-11-28T07:41:22,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:22,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:22,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-28T07:41:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-28T07:41:22,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-28T07:41:22,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1570 sec 2024-11-28T07:41:22,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.1630 sec 2024-11-28T07:41:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:23,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:41:23,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:23,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:23,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112884f1a1736aa444889662991fd69671bb_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779682432/Put/seqid=0 2024-11-28T07:41:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742050_1226 (size=14994) 2024-11-28T07:41:23,052 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:23,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112884f1a1736aa444889662991fd69671bb_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112884f1a1736aa444889662991fd69671bb_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:23,063 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/bf123bc266ac4fa1b9c3798d229f56e6, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:23,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/bf123bc266ac4fa1b9c3798d229f56e6 is 175, key is test_row_0/A:col10/1732779682432/Put/seqid=0 2024-11-28T07:41:23,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779743060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779743068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779743070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742051_1227 (size=39949) 2024-11-28T07:41:23,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/bf123bc266ac4fa1b9c3798d229f56e6 2024-11-28T07:41:23,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779743080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2c8c80d1d84b4e4d980aaee7c218bbf5 is 50, key is test_row_0/B:col10/1732779682432/Put/seqid=0 2024-11-28T07:41:23,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742052_1228 (size=12301) 2024-11-28T07:41:23,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779743180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779743180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779743390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779743390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2c8c80d1d84b4e4d980aaee7c218bbf5 2024-11-28T07:41:23,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/7929a4ba786741fc877c7d6ad1bb75de is 50, key is test_row_0/C:col10/1732779682432/Put/seqid=0 2024-11-28T07:41:23,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742053_1229 (size=12301) 2024-11-28T07:41:23,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779743580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779743590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779743700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779743700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-28T07:41:23,780 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:23,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-28T07:41:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T07:41:23,783 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:23,783 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:23,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T07:41:23,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/7929a4ba786741fc877c7d6ad1bb75de 2024-11-28T07:41:23,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T07:41:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:23,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:23,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:23,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:23,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/bf123bc266ac4fa1b9c3798d229f56e6 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bf123bc266ac4fa1b9c3798d229f56e6 2024-11-28T07:41:23,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bf123bc266ac4fa1b9c3798d229f56e6, entries=200, sequenceid=364, filesize=39.0 K 2024-11-28T07:41:23,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/2c8c80d1d84b4e4d980aaee7c218bbf5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2c8c80d1d84b4e4d980aaee7c218bbf5 2024-11-28T07:41:23,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2c8c80d1d84b4e4d980aaee7c218bbf5, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T07:41:23,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/7929a4ba786741fc877c7d6ad1bb75de as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7929a4ba786741fc877c7d6ad1bb75de 2024-11-28T07:41:23,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7929a4ba786741fc877c7d6ad1bb75de, entries=150, sequenceid=364, filesize=12.0 K 2024-11-28T07:41:23,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4e9a306a7a5304212409c030a7b68ce0 in 950ms, sequenceid=364, compaction requested=true 2024-11-28T07:41:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:23,955 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:23,955 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:23,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:23,956 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:23,956 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:23,957 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:23,957 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/47767987edd64319bdaba00f934cef65, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bf123bc266ac4fa1b9c3798d229f56e6] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=100.7 K 2024-11-28T07:41:23,957 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:23,957 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/47767987edd64319bdaba00f934cef65, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bf123bc266ac4fa1b9c3798d229f56e6] 2024-11-28T07:41:23,957 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:23,957 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:23,957 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:23,957 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4804107c944f4e889d7c08b9b8047e23, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af12ad47bec34ec0aa340f9580d967c5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2c8c80d1d84b4e4d980aaee7c218bbf5] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=36.7 K 2024-11-28T07:41:23,958 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4804107c944f4e889d7c08b9b8047e23, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732779680837 2024-11-28T07:41:23,958 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5aa4b11a8db46b0a5aefa2fe8aaf95d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732779680837 2024-11-28T07:41:23,958 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47767987edd64319bdaba00f934cef65, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732779681191 2024-11-28T07:41:23,958 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting af12ad47bec34ec0aa340f9580d967c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732779681191 2024-11-28T07:41:23,958 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf123bc266ac4fa1b9c3798d229f56e6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732779682432 2024-11-28T07:41:23,958 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c8c80d1d84b4e4d980aaee7c218bbf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732779682432 2024-11-28T07:41:23,969 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:23,970 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/af36ee8b3cbe488280555ae540363919 is 50, key is test_row_0/B:col10/1732779682432/Put/seqid=0 2024-11-28T07:41:23,972 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:23,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742054_1230 (size=13051) 2024-11-28T07:41:23,985 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128c74bcba4bf7e4440a8b9a11010ed67df_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:23,987 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128c74bcba4bf7e4440a8b9a11010ed67df_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:23,987 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c74bcba4bf7e4440a8b9a11010ed67df_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:23,991 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/af36ee8b3cbe488280555ae540363919 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af36ee8b3cbe488280555ae540363919 2024-11-28T07:41:23,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742055_1231 (size=4469) 2024-11-28T07:41:23,997 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#193 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:23,998 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/bb7499e8fdac4a86ab6ba7daa9be2714 is 175, key is test_row_0/A:col10/1732779682432/Put/seqid=0 2024-11-28T07:41:24,001 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into af36ee8b3cbe488280555ae540363919(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:24,001 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:24,001 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=13, startTime=1732779683955; duration=0sec 2024-11-28T07:41:24,001 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:24,001 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:24,001 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:24,002 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:24,002 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:24,002 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:24,002 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/464f360c148249c3a971623628b21903, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0cb01ff7e54a422da23ca2c5ca85bb3c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7929a4ba786741fc877c7d6ad1bb75de] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=36.7 K 2024-11-28T07:41:24,003 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 464f360c148249c3a971623628b21903, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1732779680837 2024-11-28T07:41:24,003 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cb01ff7e54a422da23ca2c5ca85bb3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732779681191 2024-11-28T07:41:24,004 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 7929a4ba786741fc877c7d6ad1bb75de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732779682432 2024-11-28T07:41:24,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742056_1232 (size=32005) 2024-11-28T07:41:24,023 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#194 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:24,024 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/fedd90cbc42a4ec8adce7e3eb5aa60bc is 50, key is test_row_0/C:col10/1732779682432/Put/seqid=0 2024-11-28T07:41:24,027 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/bb7499e8fdac4a86ab6ba7daa9be2714 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bb7499e8fdac4a86ab6ba7daa9be2714 2024-11-28T07:41:24,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742057_1233 (size=13051) 2024-11-28T07:41:24,036 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into bb7499e8fdac4a86ab6ba7daa9be2714(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:24,036 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:24,036 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=13, startTime=1732779683955; duration=0sec 2024-11-28T07:41:24,036 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:24,037 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:24,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-28T07:41:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:24,090 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:41:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:24,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:24,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112812680e4534584a55b19e4bc0fdc3f79c_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779683065/Put/seqid=0 2024-11-28T07:41:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T07:41:24,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742058_1234 (size=12454) 2024-11-28T07:41:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:24,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:24,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779744258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779744259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779744370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779744371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T07:41:24,442 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/fedd90cbc42a4ec8adce7e3eb5aa60bc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/fedd90cbc42a4ec8adce7e3eb5aa60bc 2024-11-28T07:41:24,447 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into fedd90cbc42a4ec8adce7e3eb5aa60bc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:24,447 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:24,447 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=13, startTime=1732779683955; duration=0sec 2024-11-28T07:41:24,447 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:24,447 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:24,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:24,508 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112812680e4534584a55b19e4bc0fdc3f79c_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112812680e4534584a55b19e4bc0fdc3f79c_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:24,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/999cc7112ec9446a9dbf3788c53fa636, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:24,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/999cc7112ec9446a9dbf3788c53fa636 is 175, key is test_row_0/A:col10/1732779683065/Put/seqid=0 2024-11-28T07:41:24,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742059_1235 (size=31255) 2024-11-28T07:41:24,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779744580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779744580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779744600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,606 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779744610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,612 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779744890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779744890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:24,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/999cc7112ec9446a9dbf3788c53fa636 2024-11-28T07:41:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T07:41:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/251a311af5a04734873b985393c150cd is 50, key is test_row_0/B:col10/1732779683065/Put/seqid=0 2024-11-28T07:41:24,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742060_1236 (size=12301) 2024-11-28T07:41:25,329 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/251a311af5a04734873b985393c150cd 2024-11-28T07:41:25,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/c3d434f5ab8142578380d0d36a91b62d is 50, key is test_row_0/C:col10/1732779683065/Put/seqid=0 2024-11-28T07:41:25,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1732779745350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:25,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742061_1237 (size=12301) 2024-11-28T07:41:25,353 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18304 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:25,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:25,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779745401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:25,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:25,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779745401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:25,753 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/c3d434f5ab8142578380d0d36a91b62d 2024-11-28T07:41:25,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/999cc7112ec9446a9dbf3788c53fa636 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/999cc7112ec9446a9dbf3788c53fa636 2024-11-28T07:41:25,764 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/999cc7112ec9446a9dbf3788c53fa636, entries=150, sequenceid=384, filesize=30.5 K 2024-11-28T07:41:25,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/251a311af5a04734873b985393c150cd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/251a311af5a04734873b985393c150cd 2024-11-28T07:41:25,770 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/251a311af5a04734873b985393c150cd, entries=150, sequenceid=384, filesize=12.0 K 2024-11-28T07:41:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/c3d434f5ab8142578380d0d36a91b62d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c3d434f5ab8142578380d0d36a91b62d 2024-11-28T07:41:25,777 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c3d434f5ab8142578380d0d36a91b62d, entries=150, sequenceid=384, filesize=12.0 K 2024-11-28T07:41:25,778 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4e9a306a7a5304212409c030a7b68ce0 in 1688ms, sequenceid=384, compaction requested=false 2024-11-28T07:41:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-28T07:41:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-28T07:41:25,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-28T07:41:25,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9970 sec 2024-11-28T07:41:25,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.0010 sec 2024-11-28T07:41:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-28T07:41:25,930 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:25,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-28T07:41:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T07:41:25,932 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:25,933 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:25,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T07:41:26,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-28T07:41:26,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,085 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:41:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:26,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285acc7d92bc034f699da75b82d70cfdf8_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779684242/Put/seqid=0 2024-11-28T07:41:26,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742062_1238 (size=12454) 2024-11-28T07:41:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,102 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411285acc7d92bc034f699da75b82d70cfdf8_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285acc7d92bc034f699da75b82d70cfdf8_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/634a14efb88d460c960e40dd928baa26, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:26,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/634a14efb88d460c960e40dd928baa26 is 175, key is test_row_0/A:col10/1732779684242/Put/seqid=0 2024-11-28T07:41:26,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742063_1239 (size=31255) 2024-11-28T07:41:26,108 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/634a14efb88d460c960e40dd928baa26 2024-11-28T07:41:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/23a7db02a6644a3b87e97b8076dcba71 is 50, key is test_row_0/B:col10/1732779684242/Put/seqid=0 2024-11-28T07:41:26,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742064_1240 (size=12301) 2024-11-28T07:41:26,121 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/23a7db02a6644a3b87e97b8076dcba71 2024-11-28T07:41:26,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/5ecd56f00e7645b7979330c47719d303 is 50, key is test_row_0/C:col10/1732779684242/Put/seqid=0 2024-11-28T07:41:26,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742065_1241 (size=12301) 2024-11-28T07:41:26,139 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/5ecd56f00e7645b7979330c47719d303 2024-11-28T07:41:26,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/634a14efb88d460c960e40dd928baa26 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/634a14efb88d460c960e40dd928baa26 2024-11-28T07:41:26,152 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/634a14efb88d460c960e40dd928baa26, entries=150, sequenceid=399, filesize=30.5 K 2024-11-28T07:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/23a7db02a6644a3b87e97b8076dcba71 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23a7db02a6644a3b87e97b8076dcba71 2024-11-28T07:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,158 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23a7db02a6644a3b87e97b8076dcba71, entries=150, sequenceid=399, filesize=12.0 K 2024-11-28T07:41:26,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/5ecd56f00e7645b7979330c47719d303 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/5ecd56f00e7645b7979330c47719d303 2024-11-28T07:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,166 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/5ecd56f00e7645b7979330c47719d303, entries=150, sequenceid=399, filesize=12.0 K 2024-11-28T07:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,167 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 4e9a306a7a5304212409c030a7b68ce0 in 81ms, sequenceid=399, compaction requested=true 2024-11-28T07:41:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-28T07:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-28T07:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-28T07:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 235 msec 2024-11-28T07:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 242 msec 2024-11-28T07:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-28T07:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,250 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-28T07:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,253 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T07:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:26,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T07:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-28T07:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:26,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-28T07:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-28T07:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-28T07:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-28T07:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 161 msec 2024-11-28T07:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:26,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:41:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:26,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289b94de78c53c421ba4059f53c16ad97c_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779686471/Put/seqid=0 2024-11-28T07:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742066_1242 (size=22618) 2024-11-28T07:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-28T07:41:26,571 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:26,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:26,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779686552, max timestamp=1732779686568Num uniques 859; 2024-11-28T07:41:26,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-28T07:41:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T07:41:26,574 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:26,574 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779686552, max timestamp=1732779686568Num uniques 859; 2024-11-28T07:41:26,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:26,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779686552, max timestamp=1732779686568Num uniques 859; 2024-11-28T07:41:26,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:26,575 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:26,575 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779746626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54956 deadline: 1732779746630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779746633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,636 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 4e9a306a7a5304212409c030a7b68ce0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779746635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T07:41:26,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:26,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779746741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779746741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779746750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:26,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T07:41:26,897 INFO [Thread-705 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-705. 2024-11-28T07:41:26,897 DEBUG [Thread-705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:26,897 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,897 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:26,902 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-28T07:41:26,902 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:26,902 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,902 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:26,903 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-28T07:41:26,903 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:26,903 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,903 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:26,908 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-28T07:41:26,908 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:26,908 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,908 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:26,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:26,911 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:26,916 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289b94de78c53c421ba4059f53c16ad97c_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289b94de78c53c421ba4059f53c16ad97c_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:26,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:26,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a is 175, key is test_row_0/A:col10/1732779686471/Put/seqid=0 2024-11-28T07:41:26,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742067_1243 (size=66023) 2024-11-28T07:41:26,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779746960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779746960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:26,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779746970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:27,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T07:41:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779747270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779747270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:27,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779747280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:27,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a 2024-11-28T07:41:27,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/d6c027007fb84c7889ee410f12e467a4 is 50, key is test_row_0/B:col10/1732779686471/Put/seqid=0 2024-11-28T07:41:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742068_1244 (size=12301) 2024-11-28T07:41:27,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:27,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:27,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T07:41:27,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/d6c027007fb84c7889ee410f12e467a4 2024-11-28T07:41:27,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/bfa0572867c94cbe8b96f425f6c91c1a is 50, key is test_row_0/C:col10/1732779686471/Put/seqid=0 2024-11-28T07:41:27,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742069_1245 (size=12301) 2024-11-28T07:41:27,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:27,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1732779747780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54938 deadline: 1732779747780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:27,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:27,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:27,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:27,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54876 deadline: 1732779747790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:27,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:41:27,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:27,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:27,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:27,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:28,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:28,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:28,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:28,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:28,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:28,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:28,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/bfa0572867c94cbe8b96f425f6c91c1a 2024-11-28T07:41:28,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a 2024-11-28T07:41:28,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a, entries=350, sequenceid=410, filesize=64.5 K 2024-11-28T07:41:28,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/d6c027007fb84c7889ee410f12e467a4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/d6c027007fb84c7889ee410f12e467a4 2024-11-28T07:41:28,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/d6c027007fb84c7889ee410f12e467a4, entries=150, sequenceid=410, filesize=12.0 K 2024-11-28T07:41:28,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/bfa0572867c94cbe8b96f425f6c91c1a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/bfa0572867c94cbe8b96f425f6c91c1a 2024-11-28T07:41:28,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/bfa0572867c94cbe8b96f425f6c91c1a, entries=150, sequenceid=410, filesize=12.0 K 2024-11-28T07:41:28,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4e9a306a7a5304212409c030a7b68ce0 in 1681ms, sequenceid=410, compaction requested=true 2024-11-28T07:41:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e9a306a7a5304212409c030a7b68ce0:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:28,163 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:41:28,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:28,163 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:41:28,164 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160538 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:41:28,164 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/A is initiating minor compaction (all files) 2024-11-28T07:41:28,164 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:41:28,164 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/A in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:28,164 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/B is initiating minor compaction (all files) 2024-11-28T07:41:28,164 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bb7499e8fdac4a86ab6ba7daa9be2714, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/999cc7112ec9446a9dbf3788c53fa636, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/634a14efb88d460c960e40dd928baa26, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=156.8 K 2024-11-28T07:41:28,164 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:28,164 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/B in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:28,164 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bb7499e8fdac4a86ab6ba7daa9be2714, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/999cc7112ec9446a9dbf3788c53fa636, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/634a14efb88d460c960e40dd928baa26, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a] 2024-11-28T07:41:28,164 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af36ee8b3cbe488280555ae540363919, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/251a311af5a04734873b985393c150cd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23a7db02a6644a3b87e97b8076dcba71, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/d6c027007fb84c7889ee410f12e467a4] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=48.8 K 2024-11-28T07:41:28,165 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7499e8fdac4a86ab6ba7daa9be2714, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732779682432 2024-11-28T07:41:28,165 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting af36ee8b3cbe488280555ae540363919, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732779682432 2024-11-28T07:41:28,165 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 999cc7112ec9446a9dbf3788c53fa636, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732779683024 2024-11-28T07:41:28,165 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 251a311af5a04734873b985393c150cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732779683024 2024-11-28T07:41:28,165 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 634a14efb88d460c960e40dd928baa26, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732779684242 2024-11-28T07:41:28,165 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a7db02a6644a3b87e97b8076dcba71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732779684242 2024-11-28T07:41:28,165 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 950eb2a53a2e48ef92d2d7e9ebcbaf1a, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732779686461 2024-11-28T07:41:28,166 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d6c027007fb84c7889ee410f12e467a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732779686471 2024-11-28T07:41:28,177 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:28,177 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#B#compaction#204 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:28,177 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/23ba904c59824b8299b9fc5ff4cd4b3b is 50, key is test_row_0/B:col10/1732779686471/Put/seqid=0 2024-11-28T07:41:28,179 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128a8ff28263c024f54af67793d6968749b_4e9a306a7a5304212409c030a7b68ce0 store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742070_1246 (size=13187) 2024-11-28T07:41:28,199 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128a8ff28263c024f54af67793d6968749b_4e9a306a7a5304212409c030a7b68ce0, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:28,199 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a8ff28263c024f54af67793d6968749b_4e9a306a7a5304212409c030a7b68ce0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:28,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742071_1247 (size=4469) 2024-11-28T07:41:28,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-28T07:41:28,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:28,260 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T07:41:28,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:28,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:28,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:28,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:28,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:28,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:28,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112882f55d63dfe34eac94841766d8fb16da_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779686624/Put/seqid=0 2024-11-28T07:41:28,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742072_1248 (size=12454) 2024-11-28T07:41:28,587 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/23ba904c59824b8299b9fc5ff4cd4b3b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23ba904c59824b8299b9fc5ff4cd4b3b 2024-11-28T07:41:28,591 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/B of 4e9a306a7a5304212409c030a7b68ce0 into 23ba904c59824b8299b9fc5ff4cd4b3b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:28,591 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:28,591 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/B, priority=12, startTime=1732779688163; duration=0sec 2024-11-28T07:41:28,591 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:28,591 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:B 2024-11-28T07:41:28,591 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:41:28,592 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:41:28,592 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 4e9a306a7a5304212409c030a7b68ce0/C is initiating minor compaction (all files) 2024-11-28T07:41:28,592 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e9a306a7a5304212409c030a7b68ce0/C in TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:28,592 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/fedd90cbc42a4ec8adce7e3eb5aa60bc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c3d434f5ab8142578380d0d36a91b62d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/5ecd56f00e7645b7979330c47719d303, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/bfa0572867c94cbe8b96f425f6c91c1a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp, totalSize=48.8 K 2024-11-28T07:41:28,593 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting fedd90cbc42a4ec8adce7e3eb5aa60bc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732779682432 2024-11-28T07:41:28,593 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting c3d434f5ab8142578380d0d36a91b62d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732779683024 2024-11-28T07:41:28,593 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ecd56f00e7645b7979330c47719d303, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1732779684242 2024-11-28T07:41:28,593 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting bfa0572867c94cbe8b96f425f6c91c1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1732779686471 2024-11-28T07:41:28,603 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#C#compaction#207 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:28,603 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/e2cf96e6c65e4ff184187405c18cb0ca is 50, key is test_row_0/C:col10/1732779686471/Put/seqid=0 2024-11-28T07:41:28,603 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e9a306a7a5304212409c030a7b68ce0#A#compaction#205 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:28,604 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/5cd1134dc32e4591ba8857fb7eab769c is 175, key is test_row_0/A:col10/1732779686471/Put/seqid=0 2024-11-28T07:41:28,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742073_1249 (size=13187) 2024-11-28T07:41:28,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742074_1250 (size=32141) 2024-11-28T07:41:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:28,679 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112882f55d63dfe34eac94841766d8fb16da_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882f55d63dfe34eac94841766d8fb16da_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:28,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6360050457274e57a025662874206569, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:28,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6360050457274e57a025662874206569 is 175, key is test_row_0/A:col10/1732779686624/Put/seqid=0 2024-11-28T07:41:28,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742075_1251 (size=31255) 2024-11-28T07:41:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T07:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:28,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. as already flushing 2024-11-28T07:41:28,792 INFO [Thread-694 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-694. 2024-11-28T07:41:28,792 INFO [Thread-702 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-702. 2024-11-28T07:41:28,792 DEBUG [Thread-694 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:28,792 DEBUG [Thread-702 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:28,792 DEBUG [Thread-694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:28,792 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:28,792 DEBUG [Thread-694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:28,792 DEBUG [Thread-702 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:28,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:28,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:28,811 INFO [Thread-698 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-698. 2024-11-28T07:41:28,812 DEBUG [Thread-698 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:28,812 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:28,812 DEBUG [Thread-698 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:28,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:29,015 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/e2cf96e6c65e4ff184187405c18cb0ca as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e2cf96e6c65e4ff184187405c18cb0ca 2024-11-28T07:41:29,019 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/5cd1134dc32e4591ba8857fb7eab769c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/5cd1134dc32e4591ba8857fb7eab769c 2024-11-28T07:41:29,019 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/C of 4e9a306a7a5304212409c030a7b68ce0 into e2cf96e6c65e4ff184187405c18cb0ca(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:29,019 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:29,019 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/C, priority=12, startTime=1732779688163; duration=0sec 2024-11-28T07:41:29,019 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:29,019 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:C 2024-11-28T07:41:29,023 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e9a306a7a5304212409c030a7b68ce0/A of 4e9a306a7a5304212409c030a7b68ce0 into 5cd1134dc32e4591ba8857fb7eab769c(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:29,023 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:29,023 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., storeName=4e9a306a7a5304212409c030a7b68ce0/A, priority=12, startTime=1732779688163; duration=0sec 2024-11-28T07:41:29,023 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:29,023 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e9a306a7a5304212409c030a7b68ce0:A 2024-11-28T07:41:29,087 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6360050457274e57a025662874206569 2024-11-28T07:41:29,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/8882e1ec8eee4ca79b0a78f61df60764 is 50, key is test_row_0/B:col10/1732779686624/Put/seqid=0 2024-11-28T07:41:29,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742076_1252 (size=12301) 2024-11-28T07:41:29,499 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/8882e1ec8eee4ca79b0a78f61df60764 2024-11-28T07:41:29,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f2c149ea2b604cb7b0296f45fb4e77ca is 50, key is test_row_0/C:col10/1732779686624/Put/seqid=0 2024-11-28T07:41:29,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742077_1253 (size=12301) 2024-11-28T07:41:29,910 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f2c149ea2b604cb7b0296f45fb4e77ca 2024-11-28T07:41:29,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/6360050457274e57a025662874206569 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6360050457274e57a025662874206569 2024-11-28T07:41:29,917 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6360050457274e57a025662874206569, entries=150, sequenceid=447, filesize=30.5 K 2024-11-28T07:41:29,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/8882e1ec8eee4ca79b0a78f61df60764 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/8882e1ec8eee4ca79b0a78f61df60764 2024-11-28T07:41:29,921 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/8882e1ec8eee4ca79b0a78f61df60764, entries=150, sequenceid=447, filesize=12.0 K 2024-11-28T07:41:29,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/f2c149ea2b604cb7b0296f45fb4e77ca as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f2c149ea2b604cb7b0296f45fb4e77ca 2024-11-28T07:41:29,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f2c149ea2b604cb7b0296f45fb4e77ca, entries=150, sequenceid=447, filesize=12.0 K 2024-11-28T07:41:29,926 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=20.13 KB/20610 for 4e9a306a7a5304212409c030a7b68ce0 in 1666ms, sequenceid=447, compaction requested=false 2024-11-28T07:41:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 4e9a306a7a5304212409c030a7b68ce0: 2024-11-28T07:41:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-28T07:41:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-28T07:41:29,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-28T07:41:29,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3520 sec 2024-11-28T07:41:29,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 3.3570 sec 2024-11-28T07:41:30,652 INFO [Thread-700 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-700. 2024-11-28T07:41:30,652 DEBUG [Thread-700 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:30,652 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:30,652 DEBUG [Thread-700 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:30,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-28T07:41:30,730 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:33,593 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T07:41:35,462 INFO [Thread-696 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-696. 2024-11-28T07:41:35,462 DEBUG [Thread-696 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:35,462 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:35,462 DEBUG [Thread-696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:35,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2356 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2292 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1066 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3196 rows 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1078 2024-11-28T07:41:35,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3227 rows 2024-11-28T07:41:35,463 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:41:35,463 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:41:35,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:35,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:35,463 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:35,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T07:41:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T07:41:35,467 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779695466"}]},"ts":"1732779695466"} 2024-11-28T07:41:35,468 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T07:41:35,468 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T07:41:35,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:41:35,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, UNASSIGN}] 2024-11-28T07:41:35,470 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, UNASSIGN 2024-11-28T07:41:35,471 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:35,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, UNASSIGN because future has completed 2024-11-28T07:41:35,472 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:41:35,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:41:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T07:41:35,625 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:35,625 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:41:35,625 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 4e9a306a7a5304212409c030a7b68ce0, disabling compactions & flushes 2024-11-28T07:41:35,625 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:35,625 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:35,625 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. after waiting 0 ms 2024-11-28T07:41:35,625 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:35,625 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 4e9a306a7a5304212409c030a7b68ce0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-28T07:41:35,626 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=A 2024-11-28T07:41:35,626 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:35,626 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=B 2024-11-28T07:41:35,626 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:35,626 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e9a306a7a5304212409c030a7b68ce0, store=C 2024-11-28T07:41:35,626 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:35,634 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287194cb91bda641f581dca6ebf2c593bc_4e9a306a7a5304212409c030a7b68ce0 is 50, key is test_row_0/A:col10/1732779688811/Put/seqid=0 2024-11-28T07:41:35,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742078_1254 (size=9914) 2024-11-28T07:41:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T07:41:36,042 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:36,046 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411287194cb91bda641f581dca6ebf2c593bc_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287194cb91bda641f581dca6ebf2c593bc_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:36,047 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8a10a76772944927bc74687fc548fa26, store: [table=TestAcidGuarantees family=A region=4e9a306a7a5304212409c030a7b68ce0] 2024-11-28T07:41:36,047 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8a10a76772944927bc74687fc548fa26 is 175, key is test_row_0/A:col10/1732779688811/Put/seqid=0 2024-11-28T07:41:36,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742079_1255 (size=22561) 2024-11-28T07:41:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T07:41:36,451 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8a10a76772944927bc74687fc548fa26 2024-11-28T07:41:36,458 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/9df3746afd374c18acbe738a3b1b48ae is 50, key is test_row_0/B:col10/1732779688811/Put/seqid=0 2024-11-28T07:41:36,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742080_1256 (size=9857) 2024-11-28T07:41:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T07:41:36,863 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/9df3746afd374c18acbe738a3b1b48ae 2024-11-28T07:41:36,870 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/1fc5a5dcc9564e6f80dbe97a28987ec0 is 50, key is test_row_0/C:col10/1732779688811/Put/seqid=0 2024-11-28T07:41:36,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742081_1257 (size=9857) 2024-11-28T07:41:37,276 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/1fc5a5dcc9564e6f80dbe97a28987ec0 2024-11-28T07:41:37,280 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/A/8a10a76772944927bc74687fc548fa26 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8a10a76772944927bc74687fc548fa26 2024-11-28T07:41:37,284 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8a10a76772944927bc74687fc548fa26, entries=100, sequenceid=458, filesize=22.0 K 2024-11-28T07:41:37,285 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/B/9df3746afd374c18acbe738a3b1b48ae as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9df3746afd374c18acbe738a3b1b48ae 2024-11-28T07:41:37,289 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9df3746afd374c18acbe738a3b1b48ae, entries=100, sequenceid=458, filesize=9.6 K 2024-11-28T07:41:37,290 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/.tmp/C/1fc5a5dcc9564e6f80dbe97a28987ec0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/1fc5a5dcc9564e6f80dbe97a28987ec0 2024-11-28T07:41:37,293 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/1fc5a5dcc9564e6f80dbe97a28987ec0, entries=100, sequenceid=458, filesize=9.6 K 2024-11-28T07:41:37,294 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4e9a306a7a5304212409c030a7b68ce0 in 1669ms, sequenceid=458, compaction requested=true 2024-11-28T07:41:37,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/555ee48a099d431c96f26ae2aff22aca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/59a3d52722014cc78cc73b2ffd543ef2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/7cfb9247828d403485e38a4023b9caa6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6c6e9f5f04684c5a8652550099b4d4e5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/89c8c04f8edd470c8c0d44b55a8801b6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/f8748bf6ff1b4e8ab0c93b116c15c9da, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6cca96400c414ba5bab4f16252f168fc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e65fbda126084ace862fc6f99e39d647, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e1843340041c414081071b99295ff32f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/2a1b30e85fb64a2ab98937b5032b7134, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/0d8b725204df44b88abfe6d08d9035fb, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/c3d330a74548455f9c58633247b9c26e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d9a6a120169b4efa82e4f66d060ef85e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d1aa02abfffd44208bd8aebf4a0825d2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/a6e1b68fce0b44bcaf1220e1650626d0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4a34c83fb0564d5e904a6554ff13bf1e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/76d71b2a5846452d89abd8fd9cf6e42b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8c522228bbbd4ce3ac0e511d5c3a676d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4ad9b142b930428dab96936b07763d12, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/47767987edd64319bdaba00f934cef65, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bf123bc266ac4fa1b9c3798d229f56e6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bb7499e8fdac4a86ab6ba7daa9be2714, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/999cc7112ec9446a9dbf3788c53fa636, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/634a14efb88d460c960e40dd928baa26, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a] to archive 2024-11-28T07:41:37,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:41:37,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/555ee48a099d431c96f26ae2aff22aca to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/555ee48a099d431c96f26ae2aff22aca 2024-11-28T07:41:37,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/59a3d52722014cc78cc73b2ffd543ef2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/59a3d52722014cc78cc73b2ffd543ef2 2024-11-28T07:41:37,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/7cfb9247828d403485e38a4023b9caa6 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/7cfb9247828d403485e38a4023b9caa6 2024-11-28T07:41:37,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6c6e9f5f04684c5a8652550099b4d4e5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6c6e9f5f04684c5a8652550099b4d4e5 2024-11-28T07:41:37,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/89c8c04f8edd470c8c0d44b55a8801b6 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/89c8c04f8edd470c8c0d44b55a8801b6 2024-11-28T07:41:37,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/f8748bf6ff1b4e8ab0c93b116c15c9da to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/f8748bf6ff1b4e8ab0c93b116c15c9da 2024-11-28T07:41:37,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6cca96400c414ba5bab4f16252f168fc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6cca96400c414ba5bab4f16252f168fc 2024-11-28T07:41:37,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e65fbda126084ace862fc6f99e39d647 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e65fbda126084ace862fc6f99e39d647 2024-11-28T07:41:37,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e1843340041c414081071b99295ff32f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/e1843340041c414081071b99295ff32f 2024-11-28T07:41:37,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/2a1b30e85fb64a2ab98937b5032b7134 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/2a1b30e85fb64a2ab98937b5032b7134 2024-11-28T07:41:37,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/0d8b725204df44b88abfe6d08d9035fb to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/0d8b725204df44b88abfe6d08d9035fb 2024-11-28T07:41:37,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/c3d330a74548455f9c58633247b9c26e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/c3d330a74548455f9c58633247b9c26e 2024-11-28T07:41:37,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d9a6a120169b4efa82e4f66d060ef85e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d9a6a120169b4efa82e4f66d060ef85e 2024-11-28T07:41:37,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d1aa02abfffd44208bd8aebf4a0825d2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/d1aa02abfffd44208bd8aebf4a0825d2 2024-11-28T07:41:37,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/a6e1b68fce0b44bcaf1220e1650626d0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/a6e1b68fce0b44bcaf1220e1650626d0 2024-11-28T07:41:37,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4a34c83fb0564d5e904a6554ff13bf1e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4a34c83fb0564d5e904a6554ff13bf1e 2024-11-28T07:41:37,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/76d71b2a5846452d89abd8fd9cf6e42b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/76d71b2a5846452d89abd8fd9cf6e42b 2024-11-28T07:41:37,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8c522228bbbd4ce3ac0e511d5c3a676d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8c522228bbbd4ce3ac0e511d5c3a676d 2024-11-28T07:41:37,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4ad9b142b930428dab96936b07763d12 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/4ad9b142b930428dab96936b07763d12 2024-11-28T07:41:37,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/b5aa4b11a8db46b0a5aefa2fe8aaf95d 2024-11-28T07:41:37,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/47767987edd64319bdaba00f934cef65 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/47767987edd64319bdaba00f934cef65 2024-11-28T07:41:37,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bf123bc266ac4fa1b9c3798d229f56e6 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bf123bc266ac4fa1b9c3798d229f56e6 2024-11-28T07:41:37,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bb7499e8fdac4a86ab6ba7daa9be2714 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/bb7499e8fdac4a86ab6ba7daa9be2714 2024-11-28T07:41:37,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/999cc7112ec9446a9dbf3788c53fa636 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/999cc7112ec9446a9dbf3788c53fa636 2024-11-28T07:41:37,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/634a14efb88d460c960e40dd928baa26 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/634a14efb88d460c960e40dd928baa26 2024-11-28T07:41:37,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/950eb2a53a2e48ef92d2d7e9ebcbaf1a 2024-11-28T07:41:37,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/6c51071ef74246cc9812ed892f0ff07d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dfeace86b5d9400ab85a6b722eb3fe67, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e6d7ea87523481b82cbf97f1fc7ecad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9cc18df9eac6489b91a8dfd6dc21c8df, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aec996170284282a1fdcf0297dd5501, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dee3d365907d4b6b947275f440a3c7be, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0623ba91a9b94614ab58fe4e19adf4d4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5989d8ee8caf45719df8fb8d6500a44c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0895bea735204ee4805fff891339acc3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/fe26e3f340c440b792443fc18d346267, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/62fbdeef020b4fabb102d59b13fce85b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5dc98729a38f41ef9113eb80893ea9c2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aa4efde72a442c49754186b2d3e8ce5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/aeba99ebd03049ba81fbc3187c47a0d3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/1c8bc51760d740e69df8cc9dc2e479d8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/ab72d7646cd64b28bb1d94547088d549, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e65a4aa51d5f4642aaede468614836a1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e93cbeed0e974f7581f0d36e23e4cb8b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4804107c944f4e889d7c08b9b8047e23, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e37f5b3cdb247a397e790a9614e971c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af12ad47bec34ec0aa340f9580d967c5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af36ee8b3cbe488280555ae540363919, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2c8c80d1d84b4e4d980aaee7c218bbf5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/251a311af5a04734873b985393c150cd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23a7db02a6644a3b87e97b8076dcba71, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/d6c027007fb84c7889ee410f12e467a4] to archive 2024-11-28T07:41:37,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:41:37,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/6c51071ef74246cc9812ed892f0ff07d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/6c51071ef74246cc9812ed892f0ff07d 2024-11-28T07:41:37,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dfeace86b5d9400ab85a6b722eb3fe67 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dfeace86b5d9400ab85a6b722eb3fe67 2024-11-28T07:41:37,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e6d7ea87523481b82cbf97f1fc7ecad to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e6d7ea87523481b82cbf97f1fc7ecad 2024-11-28T07:41:37,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9cc18df9eac6489b91a8dfd6dc21c8df to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9cc18df9eac6489b91a8dfd6dc21c8df 2024-11-28T07:41:37,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aec996170284282a1fdcf0297dd5501 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aec996170284282a1fdcf0297dd5501 2024-11-28T07:41:37,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dee3d365907d4b6b947275f440a3c7be to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/dee3d365907d4b6b947275f440a3c7be 2024-11-28T07:41:37,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0623ba91a9b94614ab58fe4e19adf4d4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0623ba91a9b94614ab58fe4e19adf4d4 2024-11-28T07:41:37,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5989d8ee8caf45719df8fb8d6500a44c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5989d8ee8caf45719df8fb8d6500a44c 2024-11-28T07:41:37,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0895bea735204ee4805fff891339acc3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/0895bea735204ee4805fff891339acc3 2024-11-28T07:41:37,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/fe26e3f340c440b792443fc18d346267 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/fe26e3f340c440b792443fc18d346267 2024-11-28T07:41:37,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/62fbdeef020b4fabb102d59b13fce85b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/62fbdeef020b4fabb102d59b13fce85b 2024-11-28T07:41:37,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5dc98729a38f41ef9113eb80893ea9c2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/5dc98729a38f41ef9113eb80893ea9c2 2024-11-28T07:41:37,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aa4efde72a442c49754186b2d3e8ce5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2aa4efde72a442c49754186b2d3e8ce5 2024-11-28T07:41:37,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/aeba99ebd03049ba81fbc3187c47a0d3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/aeba99ebd03049ba81fbc3187c47a0d3 2024-11-28T07:41:37,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/1c8bc51760d740e69df8cc9dc2e479d8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/1c8bc51760d740e69df8cc9dc2e479d8 2024-11-28T07:41:37,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/ab72d7646cd64b28bb1d94547088d549 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/ab72d7646cd64b28bb1d94547088d549 2024-11-28T07:41:37,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e65a4aa51d5f4642aaede468614836a1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e65a4aa51d5f4642aaede468614836a1 2024-11-28T07:41:37,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e93cbeed0e974f7581f0d36e23e4cb8b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/e93cbeed0e974f7581f0d36e23e4cb8b 2024-11-28T07:41:37,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4804107c944f4e889d7c08b9b8047e23 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4804107c944f4e889d7c08b9b8047e23 2024-11-28T07:41:37,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e37f5b3cdb247a397e790a9614e971c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/4e37f5b3cdb247a397e790a9614e971c 2024-11-28T07:41:37,351 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af12ad47bec34ec0aa340f9580d967c5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af12ad47bec34ec0aa340f9580d967c5 2024-11-28T07:41:37,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af36ee8b3cbe488280555ae540363919 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/af36ee8b3cbe488280555ae540363919 2024-11-28T07:41:37,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2c8c80d1d84b4e4d980aaee7c218bbf5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/2c8c80d1d84b4e4d980aaee7c218bbf5 2024-11-28T07:41:37,354 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/251a311af5a04734873b985393c150cd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/251a311af5a04734873b985393c150cd 2024-11-28T07:41:37,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23a7db02a6644a3b87e97b8076dcba71 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23a7db02a6644a3b87e97b8076dcba71 2024-11-28T07:41:37,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/d6c027007fb84c7889ee410f12e467a4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/d6c027007fb84c7889ee410f12e467a4 2024-11-28T07:41:37,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/31ae9951fed4481ebf97682afd2ecabf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/3102c0aeaf734fd18c573c5a37cf899b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/55592724e80447738258df4cab9cbc9c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7018cff82ba54b0b958cb57311da88a1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0de2f47bfba549ea8fc5f546ef0365e4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/701e236e61ae4950853935a41f7e6639, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/43c33846b9274d76a902e4fd73ab2938, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e8afdfdc9dae47bea63a593fa6968f45, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c839667856a547d89fbbf2c96b382e87, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/ef4b8c45711a45dcb1e11df2a362b421, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/52cc25d44f3c49d1821a4ad011c44112, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/a874c88352bc4c27951b2f64bd32dc25, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f42ce6d6378346328a315b07d8386cd5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/9a6837705ac946959b22174f89867323, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/d563c359092748f4ba9481203492a91e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0f6893cc3fd544a9ae82aded3686ef36, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/b00420fe26534cada8decb3b16fe4b6a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f8fad082757146f1b13a46f81f5edef9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/464f360c148249c3a971623628b21903, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/373ca903bc10442f996d1c03a57146e4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0cb01ff7e54a422da23ca2c5ca85bb3c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/fedd90cbc42a4ec8adce7e3eb5aa60bc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7929a4ba786741fc877c7d6ad1bb75de, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c3d434f5ab8142578380d0d36a91b62d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/5ecd56f00e7645b7979330c47719d303, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/bfa0572867c94cbe8b96f425f6c91c1a] to archive 2024-11-28T07:41:37,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:41:37,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/31ae9951fed4481ebf97682afd2ecabf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/31ae9951fed4481ebf97682afd2ecabf 2024-11-28T07:41:37,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/3102c0aeaf734fd18c573c5a37cf899b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/3102c0aeaf734fd18c573c5a37cf899b 2024-11-28T07:41:37,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/55592724e80447738258df4cab9cbc9c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/55592724e80447738258df4cab9cbc9c 2024-11-28T07:41:37,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7018cff82ba54b0b958cb57311da88a1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7018cff82ba54b0b958cb57311da88a1 2024-11-28T07:41:37,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0de2f47bfba549ea8fc5f546ef0365e4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0de2f47bfba549ea8fc5f546ef0365e4 2024-11-28T07:41:37,366 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/701e236e61ae4950853935a41f7e6639 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/701e236e61ae4950853935a41f7e6639 2024-11-28T07:41:37,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/43c33846b9274d76a902e4fd73ab2938 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/43c33846b9274d76a902e4fd73ab2938 2024-11-28T07:41:37,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e8afdfdc9dae47bea63a593fa6968f45 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e8afdfdc9dae47bea63a593fa6968f45 2024-11-28T07:41:37,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c839667856a547d89fbbf2c96b382e87 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c839667856a547d89fbbf2c96b382e87 2024-11-28T07:41:37,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/ef4b8c45711a45dcb1e11df2a362b421 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/ef4b8c45711a45dcb1e11df2a362b421 2024-11-28T07:41:37,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/52cc25d44f3c49d1821a4ad011c44112 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/52cc25d44f3c49d1821a4ad011c44112 2024-11-28T07:41:37,374 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/a874c88352bc4c27951b2f64bd32dc25 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/a874c88352bc4c27951b2f64bd32dc25 2024-11-28T07:41:37,375 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f42ce6d6378346328a315b07d8386cd5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f42ce6d6378346328a315b07d8386cd5 2024-11-28T07:41:37,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/9a6837705ac946959b22174f89867323 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/9a6837705ac946959b22174f89867323 2024-11-28T07:41:37,377 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/d563c359092748f4ba9481203492a91e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/d563c359092748f4ba9481203492a91e 2024-11-28T07:41:37,378 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0f6893cc3fd544a9ae82aded3686ef36 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0f6893cc3fd544a9ae82aded3686ef36 2024-11-28T07:41:37,379 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/b00420fe26534cada8decb3b16fe4b6a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/b00420fe26534cada8decb3b16fe4b6a 2024-11-28T07:41:37,380 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f8fad082757146f1b13a46f81f5edef9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f8fad082757146f1b13a46f81f5edef9 2024-11-28T07:41:37,381 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/464f360c148249c3a971623628b21903 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/464f360c148249c3a971623628b21903 2024-11-28T07:41:37,382 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/373ca903bc10442f996d1c03a57146e4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/373ca903bc10442f996d1c03a57146e4 2024-11-28T07:41:37,383 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0cb01ff7e54a422da23ca2c5ca85bb3c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/0cb01ff7e54a422da23ca2c5ca85bb3c 2024-11-28T07:41:37,384 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/fedd90cbc42a4ec8adce7e3eb5aa60bc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/fedd90cbc42a4ec8adce7e3eb5aa60bc 2024-11-28T07:41:37,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7929a4ba786741fc877c7d6ad1bb75de to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/7929a4ba786741fc877c7d6ad1bb75de 2024-11-28T07:41:37,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c3d434f5ab8142578380d0d36a91b62d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/c3d434f5ab8142578380d0d36a91b62d 2024-11-28T07:41:37,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/5ecd56f00e7645b7979330c47719d303 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/5ecd56f00e7645b7979330c47719d303 2024-11-28T07:41:37,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/bfa0572867c94cbe8b96f425f6c91c1a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/bfa0572867c94cbe8b96f425f6c91c1a 2024-11-28T07:41:37,396 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/recovered.edits/461.seqid, newMaxSeqId=461, maxSeqId=4 2024-11-28T07:41:37,397 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. 2024-11-28T07:41:37,397 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 4e9a306a7a5304212409c030a7b68ce0: Waiting for close lock at 1732779695625Running coprocessor pre-close hooks at 1732779695625Disabling compacts and flushes for region at 1732779695625Disabling writes for close at 1732779695625Obtaining lock to block concurrent updates at 1732779695625Preparing flush snapshotting stores in 4e9a306a7a5304212409c030a7b68ce0 at 1732779695625Finished memstore snapshotting TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732779695626 (+1 ms)Flushing stores of TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0. at 1732779695627 (+1 ms)Flushing 4e9a306a7a5304212409c030a7b68ce0/A: creating writer at 1732779695627Flushing mob file 4e9a306a7a5304212409c030a7b68ce0/A: appending metadata at 1732779695634 (+7 ms)Flushing mob file 4e9a306a7a5304212409c030a7b68ce0/A: closing flushed file at 1732779695634Flushing 4e9a306a7a5304212409c030a7b68ce0/A: appending metadata at 1732779696047 (+413 ms)Flushing 4e9a306a7a5304212409c030a7b68ce0/A: closing flushed file at 1732779696047Flushing 4e9a306a7a5304212409c030a7b68ce0/B: creating writer at 1732779696455 (+408 ms)Flushing 4e9a306a7a5304212409c030a7b68ce0/B: appending metadata at 1732779696458 (+3 ms)Flushing 4e9a306a7a5304212409c030a7b68ce0/B: closing flushed file at 1732779696458Flushing 4e9a306a7a5304212409c030a7b68ce0/C: creating writer at 1732779696867 (+409 ms)Flushing 4e9a306a7a5304212409c030a7b68ce0/C: appending metadata at 1732779696869 (+2 ms)Flushing 4e9a306a7a5304212409c030a7b68ce0/C: closing flushed file at 1732779696869Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@782f334e: reopening flushed file at 1732779697280 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29a67a09: reopening flushed file at 1732779697284 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c669679: reopening flushed file at 1732779697289 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 4e9a306a7a5304212409c030a7b68ce0 in 1669ms, sequenceid=458, compaction requested=true at 1732779697294 (+5 ms)Writing region close event to WAL at 1732779697392 (+98 ms)Running coprocessor post-close hooks at 1732779697396 (+4 ms)Closed at 1732779697397 (+1 ms) 2024-11-28T07:41:37,399 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,399 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=4e9a306a7a5304212409c030a7b68ce0, regionState=CLOSED 2024-11-28T07:41:37,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:41:37,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-28T07:41:37,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 4e9a306a7a5304212409c030a7b68ce0, server=f64cf8bdf19b,34437,1732779635619 in 1.9290 sec 2024-11-28T07:41:37,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-28T07:41:37,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e9a306a7a5304212409c030a7b68ce0, UNASSIGN in 1.9330 sec 2024-11-28T07:41:37,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-28T07:41:37,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9370 sec 2024-11-28T07:41:37,409 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779697409"}]},"ts":"1732779697409"} 2024-11-28T07:41:37,410 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T07:41:37,410 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T07:41:37,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9470 sec 2024-11-28T07:41:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-28T07:41:37,611 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:37,611 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T07:41:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:37,613 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T07:41:37,614 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:37,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,620 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/recovered.edits] 2024-11-28T07:41:37,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/5cd1134dc32e4591ba8857fb7eab769c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/5cd1134dc32e4591ba8857fb7eab769c 2024-11-28T07:41:37,625 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6360050457274e57a025662874206569 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/6360050457274e57a025662874206569 2024-11-28T07:41:37,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8a10a76772944927bc74687fc548fa26 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/A/8a10a76772944927bc74687fc548fa26 2024-11-28T07:41:37,629 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23ba904c59824b8299b9fc5ff4cd4b3b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/23ba904c59824b8299b9fc5ff4cd4b3b 2024-11-28T07:41:37,631 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/8882e1ec8eee4ca79b0a78f61df60764 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/8882e1ec8eee4ca79b0a78f61df60764 2024-11-28T07:41:37,632 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9df3746afd374c18acbe738a3b1b48ae to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/B/9df3746afd374c18acbe738a3b1b48ae 2024-11-28T07:41:37,635 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/1fc5a5dcc9564e6f80dbe97a28987ec0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/1fc5a5dcc9564e6f80dbe97a28987ec0 2024-11-28T07:41:37,636 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e2cf96e6c65e4ff184187405c18cb0ca to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/e2cf96e6c65e4ff184187405c18cb0ca 2024-11-28T07:41:37,638 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f2c149ea2b604cb7b0296f45fb4e77ca to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/C/f2c149ea2b604cb7b0296f45fb4e77ca 2024-11-28T07:41:37,640 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/recovered.edits/461.seqid to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0/recovered.edits/461.seqid 2024-11-28T07:41:37,641 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,641 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T07:41:37,642 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T07:41:37,643 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T07:41:37,648 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280d081c863e2344a9bca262b6e4ccd4db_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280d081c863e2344a9bca262b6e4ccd4db_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,649 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112812680e4534584a55b19e4bc0fdc3f79c_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112812680e4534584a55b19e4bc0fdc3f79c_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,650 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112835e69d410f9e4fc990fb1f746b2325da_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112835e69d410f9e4fc990fb1f746b2325da_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,652 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285acc7d92bc034f699da75b82d70cfdf8_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411285acc7d92bc034f699da75b82d70cfdf8_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,653 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287194cb91bda641f581dca6ebf2c593bc_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411287194cb91bda641f581dca6ebf2c593bc_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,654 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112873ef4efda2d24493b54180222a4583c6_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112873ef4efda2d24493b54180222a4583c6_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,655 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882f55d63dfe34eac94841766d8fb16da_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112882f55d63dfe34eac94841766d8fb16da_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,657 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883a8f9a8f50c4bec8cd5544323394a13_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883a8f9a8f50c4bec8cd5544323394a13_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,658 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883a937f73908474db57646ccd9e76b97_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112883a937f73908474db57646ccd9e76b97_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,659 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112884f1a1736aa444889662991fd69671bb_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112884f1a1736aa444889662991fd69671bb_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,660 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128964434f71a80492e8cca1b37952f5d3a_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128964434f71a80492e8cca1b37952f5d3a_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,661 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112896e6ce2db0f24be588346c88746644a6_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112896e6ce2db0f24be588346c88746644a6_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,662 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128983e5deaf91c4fd5889a471971ddeba1_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128983e5deaf91c4fd5889a471971ddeba1_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,664 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289b94de78c53c421ba4059f53c16ad97c_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289b94de78c53c421ba4059f53c16ad97c_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,665 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b320e5dade3c4b2384a468400d0b3e88_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b320e5dade3c4b2384a468400d0b3e88_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,666 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b78a6884b72945719cd2214eba0c14ad_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128b78a6884b72945719cd2214eba0c14ad_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,667 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ca529c6d776341d89046f7714eba99bb_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128ca529c6d776341d89046f7714eba99bb_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,669 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dac6f7a14e80464aa66d4d181eb67c42_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128dac6f7a14e80464aa66d4d181eb67c42_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,670 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e185e97806924fb8a6780ca1501c840a_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e185e97806924fb8a6780ca1501c840a_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,671 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e67636ed5aec43c9b9b14093a02a4a2c_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e67636ed5aec43c9b9b14093a02a4a2c_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,674 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f266ae8b20d840b2b17a2b5ce1a2d110_4e9a306a7a5304212409c030a7b68ce0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f266ae8b20d840b2b17a2b5ce1a2d110_4e9a306a7a5304212409c030a7b68ce0 2024-11-28T07:41:37,674 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T07:41:37,677 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:37,679 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T07:41:37,681 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T07:41:37,682 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:37,682 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T07:41:37,683 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732779697682"}]},"ts":"9223372036854775807"} 2024-11-28T07:41:37,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T07:41:37,685 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4e9a306a7a5304212409c030a7b68ce0, NAME => 'TestAcidGuarantees,,1732779663581.4e9a306a7a5304212409c030a7b68ce0.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T07:41:37,685 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T07:41:37,685 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732779697685"}]},"ts":"9223372036854775807"} 2024-11-28T07:41:37,687 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T07:41:37,688 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:37,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-28T07:41:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-28T07:41:37,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T07:41:37,721 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:37,734 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2099947969_22 at /127.0.0.1:51074 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 369) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7078 (was 7169) 2024-11-28T07:41:37,745 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=7077 2024-11-28T07:41:37,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:41:37,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:41:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T07:41:37,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:41:37,749 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:37,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-28T07:41:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-28T07:41:37,750 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:41:37,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742082_1258 (size=1061) 2024-11-28T07:41:37,758 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:41:37,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742083_1259 (size=53) 2024-11-28T07:41:37,767 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:41:37,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ebed6cd25868ca618a5fe2887edb3192, disabling compactions & flushes 2024-11-28T07:41:37,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:37,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:37,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. after waiting 0 ms 2024-11-28T07:41:37,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:37,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:37,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ebed6cd25868ca618a5fe2887edb3192: Waiting for close lock at 1732779697768Disabling compacts and flushes for region at 1732779697768Disabling writes for close at 1732779697768Writing region close event to WAL at 1732779697768Closed at 1732779697768 2024-11-28T07:41:37,769 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:41:37,769 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732779697769"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779697769"}]},"ts":"1732779697769"} 2024-11-28T07:41:37,771 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:41:37,772 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:41:37,772 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779697772"}]},"ts":"1732779697772"} 2024-11-28T07:41:37,773 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T07:41:37,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, ASSIGN}] 2024-11-28T07:41:37,774 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, ASSIGN 2024-11-28T07:41:37,775 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=false 2024-11-28T07:41:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-28T07:41:37,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ebed6cd25868ca618a5fe2887edb3192, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:37,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, ASSIGN because future has completed 2024-11-28T07:41:37,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:41:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-28T07:41:38,082 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,083 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:41:38,083 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,083 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:41:38,083 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,083 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,084 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,086 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:38,086 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebed6cd25868ca618a5fe2887edb3192 columnFamilyName A 2024-11-28T07:41:38,086 DEBUG [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:38,087 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.HStore(327): Store=ebed6cd25868ca618a5fe2887edb3192/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:38,087 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,088 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:38,088 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebed6cd25868ca618a5fe2887edb3192 columnFamilyName B 2024-11-28T07:41:38,088 DEBUG [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:38,089 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.HStore(327): Store=ebed6cd25868ca618a5fe2887edb3192/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:38,089 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,089 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:41:38,090 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebed6cd25868ca618a5fe2887edb3192 columnFamilyName C 2024-11-28T07:41:38,090 DEBUG [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:41:38,090 INFO [StoreOpener-ebed6cd25868ca618a5fe2887edb3192-1 {}] regionserver.HStore(327): Store=ebed6cd25868ca618a5fe2887edb3192/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:41:38,090 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,090 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,091 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,091 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,092 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,092 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,092 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:41:38,093 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,095 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:41:38,095 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened ebed6cd25868ca618a5fe2887edb3192; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70743607, jitterRate=0.05416189134120941}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:41:38,096 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,097 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for ebed6cd25868ca618a5fe2887edb3192: Running coprocessor pre-open hook at 1732779698083Writing region info on filesystem at 1732779698083Initializing all the Stores at 1732779698084 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779698084Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779698084Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779698084Cleaning up temporary data from old regions at 1732779698092 (+8 ms)Running coprocessor post-open hooks at 1732779698096 (+4 ms)Region opened successfully at 1732779698097 (+1 ms) 2024-11-28T07:41:38,098 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., pid=67, masterSystemTime=1732779698079 2024-11-28T07:41:38,100 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,100 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,100 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ebed6cd25868ca618a5fe2887edb3192, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:41:38,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-28T07:41:38,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 in 174 msec 2024-11-28T07:41:38,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-28T07:41:38,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, ASSIGN in 331 msec 2024-11-28T07:41:38,106 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:41:38,107 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779698106"}]},"ts":"1732779698106"} 2024-11-28T07:41:38,108 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T07:41:38,109 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:41:38,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 363 msec 2024-11-28T07:41:38,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-28T07:41:38,381 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:38,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,382 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,382 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,383 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67aa10a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66db35c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,386 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,390 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:41:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,39063,1732779634677 2024-11-28T07:41:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3903852d 2024-11-28T07:41:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:41:38,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:41:38,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,394 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,394 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de58590, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667f1f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,399 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,399 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3360de83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135a4636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,402 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,403 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240437cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57955860, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,407 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa7a6ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d7c42f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,411 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20dfce25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78200be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,418 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6597df09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,420 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@413706c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,422 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@513ebb3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,423 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4bebd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,425 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cbf99e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,427 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b40e651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,428 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,428 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,429 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23bf3ee5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,431 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478f6b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:41:38,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:41:38,437 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:41:38,437 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:41:38,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:41:38,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764c0fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:41:38,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:41:38,438 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:38,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:41:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330ca02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:41:38,440 DEBUG [Thread-1198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,440 DEBUG [Thread-1213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,441 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,441 DEBUG [Thread-1213 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,441 DEBUG [Thread-1198 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,441 DEBUG [Thread-1213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,442 DEBUG [Thread-1198 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,449 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:38,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-28T07:41:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T07:41:38,451 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:38,452 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:38,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:38,453 DEBUG [Thread-1207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,454 DEBUG [Thread-1207 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,455 DEBUG [Thread-1207 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,457 DEBUG [Thread-1211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:38,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:41:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:38,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:38,467 DEBUG [Thread-1205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,481 DEBUG [Thread-1194 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,497 DEBUG [Thread-1209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,501 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,519 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:41:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:41:38,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:41:38,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:41:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:41:38,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4e4833e209804bd28f8e5c29ca13eef3 is 50, key is test_row_0/A:col10/1732779698461/Put/seqid=0 2024-11-28T07:41:38,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779698495, max timestamp=1732779698527Num uniques 859; 2024-11-28T07:41:38,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:38,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779698495, max timestamp=1732779698527Num uniques 859; 2024-11-28T07:41:38,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:38,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779698495, max timestamp=1732779698527Num uniques 859; 2024-11-28T07:41:38,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:38,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742084_1260 (size=9657) 2024-11-28T07:41:38,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4e4833e209804bd28f8e5c29ca13eef3 2024-11-28T07:41:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T07:41:38,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/7771e1d42c2a415d8b85d66b331ff791 is 50, key is test_row_0/B:col10/1732779698461/Put/seqid=0 2024-11-28T07:41:38,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742085_1261 (size=9657) 2024-11-28T07:41:38,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T07:41:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779758571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779758571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779758633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779758633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779758644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779758741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779758750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779758750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779758750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T07:41:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779758761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T07:41:38,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T07:41:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:38,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:38,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779758950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779758960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779758961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779758961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:38,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779758970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:38,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/7771e1d42c2a415d8b85d66b331ff791 2024-11-28T07:41:39,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/6268725cc5834b48b1997dcc17a3097b is 50, key is test_row_0/C:col10/1732779698461/Put/seqid=0 2024-11-28T07:41:39,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742086_1262 (size=9657) 2024-11-28T07:41:39,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T07:41:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:39,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T07:41:39,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T07:41:39,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:39,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:39,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:39,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779759260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779759270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779759270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779759270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779759281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T07:41:39,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:39,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:39,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/6268725cc5834b48b1997dcc17a3097b 2024-11-28T07:41:39,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4e4833e209804bd28f8e5c29ca13eef3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4e4833e209804bd28f8e5c29ca13eef3 2024-11-28T07:41:39,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4e4833e209804bd28f8e5c29ca13eef3, entries=100, sequenceid=11, filesize=9.4 K 2024-11-28T07:41:39,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/7771e1d42c2a415d8b85d66b331ff791 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7771e1d42c2a415d8b85d66b331ff791 2024-11-28T07:41:39,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7771e1d42c2a415d8b85d66b331ff791, entries=100, sequenceid=11, filesize=9.4 K 2024-11-28T07:41:39,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/6268725cc5834b48b1997dcc17a3097b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6268725cc5834b48b1997dcc17a3097b 2024-11-28T07:41:39,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6268725cc5834b48b1997dcc17a3097b, entries=100, sequenceid=11, filesize=9.4 K 2024-11-28T07:41:39,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ebed6cd25868ca618a5fe2887edb3192 in 1003ms, sequenceid=11, compaction requested=false 2024-11-28T07:41:39,465 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T07:41:39,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:39,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-28T07:41:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:39,527 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-28T07:41:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/eab4da29af4b4d298f992c0d07a343c9 is 50, key is test_row_0/A:col10/1732779698586/Put/seqid=0 2024-11-28T07:41:39,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742087_1263 (size=12001) 2024-11-28T07:41:39,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/eab4da29af4b4d298f992c0d07a343c9 2024-11-28T07:41:39,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/dffc6ee655b94bc79f2df590b8c3d9dc is 50, key is test_row_0/B:col10/1732779698586/Put/seqid=0 2024-11-28T07:41:39,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742088_1264 (size=12001) 2024-11-28T07:41:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T07:41:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:39,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:39,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779759786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779759785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779759790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779759790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779759790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779759900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779759901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779759901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779759901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:39,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:39,967 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/dffc6ee655b94bc79f2df590b8c3d9dc 2024-11-28T07:41:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/4f1c169e95ce4d2ca5a80acf5b59b101 is 50, key is test_row_0/C:col10/1732779698586/Put/seqid=0 2024-11-28T07:41:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742089_1265 (size=12001) 2024-11-28T07:41:40,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779760110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779760110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779760111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779760111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,381 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/4f1c169e95ce4d2ca5a80acf5b59b101 2024-11-28T07:41:40,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/eab4da29af4b4d298f992c0d07a343c9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/eab4da29af4b4d298f992c0d07a343c9 2024-11-28T07:41:40,391 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/eab4da29af4b4d298f992c0d07a343c9, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T07:41:40,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/dffc6ee655b94bc79f2df590b8c3d9dc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/dffc6ee655b94bc79f2df590b8c3d9dc 2024-11-28T07:41:40,397 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/dffc6ee655b94bc79f2df590b8c3d9dc, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T07:41:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/4f1c169e95ce4d2ca5a80acf5b59b101 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/4f1c169e95ce4d2ca5a80acf5b59b101 2024-11-28T07:41:40,402 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/4f1c169e95ce4d2ca5a80acf5b59b101, entries=150, sequenceid=48, filesize=11.7 K 2024-11-28T07:41:40,403 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for ebed6cd25868ca618a5fe2887edb3192 in 876ms, sequenceid=48, compaction requested=false 2024-11-28T07:41:40,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:40,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:40,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-28T07:41:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-28T07:41:40,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-28T07:41:40,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9520 sec 2024-11-28T07:41:40,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9580 sec 2024-11-28T07:41:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:40,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:41:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:40,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:40,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/ad9f2632ebde45cb9987c55883c06cb4 is 50, key is test_row_0/A:col10/1732779700421/Put/seqid=0 2024-11-28T07:41:40,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742090_1266 (size=14341) 2024-11-28T07:41:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779760466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779760471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779760471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779760472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779760580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779760580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779760580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779760580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-28T07:41:40,601 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:40,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-28T07:41:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T07:41:40,603 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:40,604 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:40,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T07:41:40,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:40,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:40,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:40,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:40,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:40,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:40,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779760790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779760790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779760790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779760790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779760810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:40,813 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:40,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:40,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/ad9f2632ebde45cb9987c55883c06cb4 2024-11-28T07:41:40,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/4539013062e84f8f8f0a04d3f2516589 is 50, key is test_row_0/B:col10/1732779700421/Put/seqid=0 2024-11-28T07:41:40,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742091_1267 (size=12001) 2024-11-28T07:41:40,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:40,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:40,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T07:41:41,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:41,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:41,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779761100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779761100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779761100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779761101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:41,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:41,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T07:41:41,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/4539013062e84f8f8f0a04d3f2516589 2024-11-28T07:41:41,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/f70e3236cd5c4be286642b84acea8c8e is 50, key is test_row_0/C:col10/1732779700421/Put/seqid=0 2024-11-28T07:41:41,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742092_1268 (size=12001) 2024-11-28T07:41:41,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:41,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:41,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779761610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779761610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779761610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779761610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:41,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/f70e3236cd5c4be286642b84acea8c8e 2024-11-28T07:41:41,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/ad9f2632ebde45cb9987c55883c06cb4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/ad9f2632ebde45cb9987c55883c06cb4 2024-11-28T07:41:41,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/ad9f2632ebde45cb9987c55883c06cb4, entries=200, sequenceid=62, filesize=14.0 K 2024-11-28T07:41:41,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/4539013062e84f8f8f0a04d3f2516589 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/4539013062e84f8f8f0a04d3f2516589 2024-11-28T07:41:41,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/4539013062e84f8f8f0a04d3f2516589, entries=150, sequenceid=62, filesize=11.7 K 2024-11-28T07:41:41,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/f70e3236cd5c4be286642b84acea8c8e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f70e3236cd5c4be286642b84acea8c8e 2024-11-28T07:41:41,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:41,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f70e3236cd5c4be286642b84acea8c8e, entries=150, sequenceid=62, filesize=11.7 K 2024-11-28T07:41:41,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ebed6cd25868ca618a5fe2887edb3192 in 1266ms, sequenceid=62, compaction requested=true 2024-11-28T07:41:41,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:41,689 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:41,689 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:41,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:41,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:41,690 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:41,690 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:41,690 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,690 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4e4833e209804bd28f8e5c29ca13eef3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/eab4da29af4b4d298f992c0d07a343c9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/ad9f2632ebde45cb9987c55883c06cb4] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=35.2 K 2024-11-28T07:41:41,690 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4833e209804bd28f8e5c29ca13eef3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732779698461 2024-11-28T07:41:41,691 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting eab4da29af4b4d298f992c0d07a343c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732779698553 2024-11-28T07:41:41,691 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9f2632ebde45cb9987c55883c06cb4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779699788 2024-11-28T07:41:41,692 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:41,692 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:41,692 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,692 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7771e1d42c2a415d8b85d66b331ff791, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/dffc6ee655b94bc79f2df590b8c3d9dc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/4539013062e84f8f8f0a04d3f2516589] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=32.9 K 2024-11-28T07:41:41,693 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 7771e1d42c2a415d8b85d66b331ff791, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732779698461 2024-11-28T07:41:41,693 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting dffc6ee655b94bc79f2df590b8c3d9dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732779698553 2024-11-28T07:41:41,693 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4539013062e84f8f8f0a04d3f2516589, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779700421 2024-11-28T07:41:41,701 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:41,702 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4988ef0ab23e47bebe233fe4b4f9b102 is 50, key is test_row_0/A:col10/1732779700421/Put/seqid=0 2024-11-28T07:41:41,702 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:41,703 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/cc7e4bfc8ba040b78e5af00b616fd611 is 50, key is test_row_0/B:col10/1732779700421/Put/seqid=0 2024-11-28T07:41:41,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742093_1269 (size=12104) 2024-11-28T07:41:41,721 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4988ef0ab23e47bebe233fe4b4f9b102 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4988ef0ab23e47bebe233fe4b4f9b102 2024-11-28T07:41:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742094_1270 (size=12104) 2024-11-28T07:41:41,729 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 4988ef0ab23e47bebe233fe4b4f9b102(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:41,729 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:41,729 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779701688; duration=0sec 2024-11-28T07:41:41,729 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:41,729 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:41,729 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:41,730 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:41,730 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:41,731 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,731 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6268725cc5834b48b1997dcc17a3097b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/4f1c169e95ce4d2ca5a80acf5b59b101, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f70e3236cd5c4be286642b84acea8c8e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=32.9 K 2024-11-28T07:41:41,732 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6268725cc5834b48b1997dcc17a3097b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732779698461 2024-11-28T07:41:41,732 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f1c169e95ce4d2ca5a80acf5b59b101, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732779698553 2024-11-28T07:41:41,733 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting f70e3236cd5c4be286642b84acea8c8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779700421 2024-11-28T07:41:41,738 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/cc7e4bfc8ba040b78e5af00b616fd611 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/cc7e4bfc8ba040b78e5af00b616fd611 2024-11-28T07:41:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T07:41:41,745 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into cc7e4bfc8ba040b78e5af00b616fd611(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:41,745 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:41,745 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779701689; duration=0sec 2024-11-28T07:41:41,745 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:41,745 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:41,748 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:41,749 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/04dff67ca8bf426c891f1935b28e0f8a is 50, key is test_row_0/C:col10/1732779700421/Put/seqid=0 2024-11-28T07:41:41,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742095_1271 (size=12104) 2024-11-28T07:41:41,763 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/04dff67ca8bf426c891f1935b28e0f8a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04dff67ca8bf426c891f1935b28e0f8a 2024-11-28T07:41:41,770 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into 04dff67ca8bf426c891f1935b28e0f8a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:41,770 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:41,770 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779701689; duration=0sec 2024-11-28T07:41:41,770 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:41,770 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:41,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-28T07:41:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,836 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:41:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:41,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/c09fee080dd34ce7b85e020f62f33af1 is 50, key is test_row_0/A:col10/1732779700470/Put/seqid=0 2024-11-28T07:41:41,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742096_1272 (size=12001) 2024-11-28T07:41:41,851 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/c09fee080dd34ce7b85e020f62f33af1 2024-11-28T07:41:41,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/d2ee3dc53763488e803035ea7baba136 is 50, key is test_row_0/B:col10/1732779700470/Put/seqid=0 2024-11-28T07:41:41,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742097_1273 (size=12001) 2024-11-28T07:41:41,880 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/d2ee3dc53763488e803035ea7baba136 2024-11-28T07:41:41,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/29d229eb8f994aacb9fe3585eeb9255a is 50, key is test_row_0/C:col10/1732779700470/Put/seqid=0 2024-11-28T07:41:41,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742098_1274 (size=12001) 2024-11-28T07:41:41,915 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/29d229eb8f994aacb9fe3585eeb9255a 2024-11-28T07:41:41,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/c09fee080dd34ce7b85e020f62f33af1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c09fee080dd34ce7b85e020f62f33af1 2024-11-28T07:41:41,927 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c09fee080dd34ce7b85e020f62f33af1, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T07:41:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/d2ee3dc53763488e803035ea7baba136 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d2ee3dc53763488e803035ea7baba136 2024-11-28T07:41:41,932 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d2ee3dc53763488e803035ea7baba136, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T07:41:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/29d229eb8f994aacb9fe3585eeb9255a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/29d229eb8f994aacb9fe3585eeb9255a 2024-11-28T07:41:41,939 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/29d229eb8f994aacb9fe3585eeb9255a, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T07:41:41,940 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ebed6cd25868ca618a5fe2887edb3192 in 104ms, sequenceid=83, compaction requested=false 2024-11-28T07:41:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-28T07:41:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-28T07:41:41,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-28T07:41:41,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3370 sec 2024-11-28T07:41:41,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.3450 sec 2024-11-28T07:41:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:42,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:41:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:42,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/f20207bbe19248139de7ca7e95c2da17 is 50, key is test_row_0/A:col10/1732779702628/Put/seqid=0 2024-11-28T07:41:42,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742099_1275 (size=16681) 2024-11-28T07:41:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779762652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779762654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779702645, max timestamp=1732779702654Num uniques 859; 2024-11-28T07:41:42,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:41:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779702645, max timestamp=1732779702654Num uniques 859; 2024-11-28T07:41:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779702645, max timestamp=1732779702654Num uniques 859; 2024-11-28T07:41:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:41:42,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:41:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779762706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779762710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-28T07:41:42,761 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:42,761 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-28T07:41:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T07:41:42,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:42,764 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:42,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:42,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779762760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779762770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779762820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779762820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779762821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,824 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T07:41:42,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T07:41:42,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:42,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:42,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:42,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:42,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:42,927 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:41:42,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779762970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:42,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779762980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779763030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779763030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/f20207bbe19248139de7ca7e95c2da17 2024-11-28T07:41:43,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/363f5411a8f541378f5cb4ce754db70f is 50, key is test_row_0/B:col10/1732779702628/Put/seqid=0 2024-11-28T07:41:43,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742100_1276 (size=12001) 2024-11-28T07:41:43,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T07:41:43,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T07:41:43,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T07:41:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779763280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779763290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779763340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779763340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T07:41:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T07:41:43,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/363f5411a8f541378f5cb4ce754db70f 2024-11-28T07:41:43,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/a1d2ad410d0a411f9aa854db326ef64a is 50, key is test_row_0/C:col10/1732779702628/Put/seqid=0 2024-11-28T07:41:43,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742101_1277 (size=12001) 2024-11-28T07:41:43,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/a1d2ad410d0a411f9aa854db326ef64a 2024-11-28T07:41:43,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/f20207bbe19248139de7ca7e95c2da17 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f20207bbe19248139de7ca7e95c2da17 2024-11-28T07:41:43,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f20207bbe19248139de7ca7e95c2da17, entries=250, sequenceid=94, filesize=16.3 K 2024-11-28T07:41:43,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/363f5411a8f541378f5cb4ce754db70f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/363f5411a8f541378f5cb4ce754db70f 2024-11-28T07:41:43,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/363f5411a8f541378f5cb4ce754db70f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T07:41:43,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/a1d2ad410d0a411f9aa854db326ef64a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a1d2ad410d0a411f9aa854db326ef64a 2024-11-28T07:41:43,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a1d2ad410d0a411f9aa854db326ef64a, entries=150, sequenceid=94, filesize=11.7 K 2024-11-28T07:41:43,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ebed6cd25868ca618a5fe2887edb3192 in 856ms, sequenceid=94, compaction requested=true 2024-11-28T07:41:43,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:43,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:43,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:43,486 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:43,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:43,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:43,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:43,486 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:43,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:43,487 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:43,487 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:43,487 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:43,487 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:43,487 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,487 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,487 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4988ef0ab23e47bebe233fe4b4f9b102, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c09fee080dd34ce7b85e020f62f33af1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f20207bbe19248139de7ca7e95c2da17] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=39.8 K 2024-11-28T07:41:43,487 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/cc7e4bfc8ba040b78e5af00b616fd611, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d2ee3dc53763488e803035ea7baba136, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/363f5411a8f541378f5cb4ce754db70f] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=35.3 K 2024-11-28T07:41:43,488 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4988ef0ab23e47bebe233fe4b4f9b102, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779700421 2024-11-28T07:41:43,488 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting cc7e4bfc8ba040b78e5af00b616fd611, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779700421 2024-11-28T07:41:43,488 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c09fee080dd34ce7b85e020f62f33af1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779700435 2024-11-28T07:41:43,488 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ee3dc53763488e803035ea7baba136, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779700435 2024-11-28T07:41:43,489 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 363f5411a8f541378f5cb4ce754db70f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779702622 2024-11-28T07:41:43,489 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting f20207bbe19248139de7ca7e95c2da17, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779702622 2024-11-28T07:41:43,497 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:43,497 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:43,497 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/002c0fb24d494173970f16971a74457f is 50, key is test_row_0/A:col10/1732779702628/Put/seqid=0 2024-11-28T07:41:43,498 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/350b43cb63314543a44aba11d876d7f9 is 50, key is test_row_0/B:col10/1732779702628/Put/seqid=0 2024-11-28T07:41:43,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742102_1278 (size=12207) 2024-11-28T07:41:43,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742103_1279 (size=12207) 2024-11-28T07:41:43,526 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/002c0fb24d494173970f16971a74457f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/002c0fb24d494173970f16971a74457f 2024-11-28T07:41:43,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-28T07:41:43,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,530 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-28T07:41:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:43,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:43,532 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/350b43cb63314543a44aba11d876d7f9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/350b43cb63314543a44aba11d876d7f9 2024-11-28T07:41:43,535 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 002c0fb24d494173970f16971a74457f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:43,535 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:43,535 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779703485; duration=0sec 2024-11-28T07:41:43,535 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:43,535 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:43,535 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:43,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/503dc3c6306349f1a9c7ed1bd6aa90fc is 50, key is test_row_0/A:col10/1732779702672/Put/seqid=0 2024-11-28T07:41:43,537 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:43,537 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:43,537 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:43,537 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04dff67ca8bf426c891f1935b28e0f8a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/29d229eb8f994aacb9fe3585eeb9255a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a1d2ad410d0a411f9aa854db326ef64a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=35.3 K 2024-11-28T07:41:43,537 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04dff67ca8bf426c891f1935b28e0f8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779700421 2024-11-28T07:41:43,538 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d229eb8f994aacb9fe3585eeb9255a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779700435 2024-11-28T07:41:43,538 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d2ad410d0a411f9aa854db326ef64a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779702622 2024-11-28T07:41:43,540 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into 350b43cb63314543a44aba11d876d7f9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:43,540 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:43,540 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779703486; duration=0sec 2024-11-28T07:41:43,540 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:43,540 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:43,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742104_1280 (size=12001) 2024-11-28T07:41:43,554 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:43,554 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/3421029670244fd58bacf51d9ee0c33c is 50, key is test_row_0/C:col10/1732779702628/Put/seqid=0 2024-11-28T07:41:43,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742105_1281 (size=12207) 2024-11-28T07:41:43,586 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/3421029670244fd58bacf51d9ee0c33c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3421029670244fd58bacf51d9ee0c33c 2024-11-28T07:41:43,592 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into 3421029670244fd58bacf51d9ee0c33c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:43,592 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:43,592 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779703486; duration=0sec 2024-11-28T07:41:43,592 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:43,592 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:43,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:43,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:43,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779763811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779763813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779763850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779763850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T07:41:43,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779763920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779763920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:43,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:43,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/503dc3c6306349f1a9c7ed1bd6aa90fc 2024-11-28T07:41:43,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/315b7ed13fa1439185286f844dbe637f is 50, key is test_row_0/B:col10/1732779702672/Put/seqid=0 2024-11-28T07:41:43,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742106_1282 (size=12001) 2024-11-28T07:41:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779764130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779764130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:44,367 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/315b7ed13fa1439185286f844dbe637f 2024-11-28T07:41:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b0fade93e97449548cbaa1d0c98718be is 50, key is test_row_0/C:col10/1732779702672/Put/seqid=0 2024-11-28T07:41:44,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742107_1283 (size=12001) 2024-11-28T07:41:44,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779764440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779764440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:44,792 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b0fade93e97449548cbaa1d0c98718be 2024-11-28T07:41:44,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/503dc3c6306349f1a9c7ed1bd6aa90fc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/503dc3c6306349f1a9c7ed1bd6aa90fc 2024-11-28T07:41:44,803 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/503dc3c6306349f1a9c7ed1bd6aa90fc, entries=150, sequenceid=128, filesize=11.7 K 2024-11-28T07:41:44,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/315b7ed13fa1439185286f844dbe637f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/315b7ed13fa1439185286f844dbe637f 2024-11-28T07:41:44,809 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/315b7ed13fa1439185286f844dbe637f, entries=150, sequenceid=128, filesize=11.7 K 2024-11-28T07:41:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b0fade93e97449548cbaa1d0c98718be as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b0fade93e97449548cbaa1d0c98718be 2024-11-28T07:41:44,814 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b0fade93e97449548cbaa1d0c98718be, entries=150, sequenceid=128, filesize=11.7 K 2024-11-28T07:41:44,815 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for ebed6cd25868ca618a5fe2887edb3192 in 1285ms, sequenceid=128, compaction requested=false 2024-11-28T07:41:44,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:44,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:44,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-28T07:41:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-28T07:41:44,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-28T07:41:44,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0530 sec 2024-11-28T07:41:44,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0590 sec 2024-11-28T07:41:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:44,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:41:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:44,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:44,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/a3aafd3218ac48f6b750814b60d0eaa2 is 50, key is test_row_0/A:col10/1732779704862/Put/seqid=0 2024-11-28T07:41:44,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742108_1284 (size=16931) 2024-11-28T07:41:44,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/a3aafd3218ac48f6b750814b60d0eaa2 2024-11-28T07:41:44,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/42921e884f1547eda7aa34f5621ef8bc is 50, key is test_row_0/B:col10/1732779704862/Put/seqid=0 2024-11-28T07:41:44,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742109_1285 (size=12151) 2024-11-28T07:41:44,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/42921e884f1547eda7aa34f5621ef8bc 2024-11-28T07:41:44,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/6d1e5217d73049a8b8dd56ea09ff353b is 50, key is test_row_0/C:col10/1732779704862/Put/seqid=0 2024-11-28T07:41:44,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742110_1286 (size=12151) 2024-11-28T07:41:44,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-28T07:41:44,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:44,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:44,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779764906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-28T07:41:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:44,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T07:41:44,913 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:44,915 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:44,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:44,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779764913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:44,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779764950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:44,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779764955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T07:41:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779765021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779765021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T07:41:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:45,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T07:41:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:45,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T07:41:45,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779765231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779765231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/6d1e5217d73049a8b8dd56ea09ff353b 2024-11-28T07:41:45,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/a3aafd3218ac48f6b750814b60d0eaa2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a3aafd3218ac48f6b750814b60d0eaa2 2024-11-28T07:41:45,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a3aafd3218ac48f6b750814b60d0eaa2, entries=250, sequenceid=144, filesize=16.5 K 2024-11-28T07:41:45,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/42921e884f1547eda7aa34f5621ef8bc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/42921e884f1547eda7aa34f5621ef8bc 2024-11-28T07:41:45,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/42921e884f1547eda7aa34f5621ef8bc, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T07:41:45,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/6d1e5217d73049a8b8dd56ea09ff353b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6d1e5217d73049a8b8dd56ea09ff353b 2024-11-28T07:41:45,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6d1e5217d73049a8b8dd56ea09ff353b, entries=150, sequenceid=144, filesize=11.9 K 2024-11-28T07:41:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ebed6cd25868ca618a5fe2887edb3192 in 463ms, sequenceid=144, compaction requested=true 2024-11-28T07:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:45,329 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:45,329 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:45,330 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:45,330 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:45,330 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,330 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/350b43cb63314543a44aba11d876d7f9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/315b7ed13fa1439185286f844dbe637f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/42921e884f1547eda7aa34f5621ef8bc] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=35.5 K 2024-11-28T07:41:45,331 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41139 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:45,331 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 350b43cb63314543a44aba11d876d7f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779702622 2024-11-28T07:41:45,331 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:45,331 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,331 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/002c0fb24d494173970f16971a74457f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/503dc3c6306349f1a9c7ed1bd6aa90fc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a3aafd3218ac48f6b750814b60d0eaa2] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=40.2 K 2024-11-28T07:41:45,331 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 315b7ed13fa1439185286f844dbe637f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732779702672 2024-11-28T07:41:45,331 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 002c0fb24d494173970f16971a74457f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779702622 2024-11-28T07:41:45,332 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 42921e884f1547eda7aa34f5621ef8bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732779703802 2024-11-28T07:41:45,332 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503dc3c6306349f1a9c7ed1bd6aa90fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732779702672 2024-11-28T07:41:45,332 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3aafd3218ac48f6b750814b60d0eaa2, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732779703802 2024-11-28T07:41:45,339 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:45,340 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/542a129efad94ad8a325643bcd255443 is 50, key is test_row_0/B:col10/1732779704862/Put/seqid=0 2024-11-28T07:41:45,341 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:45,341 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/2506cb48f791468d8571deb6002f1813 is 50, key is test_row_0/A:col10/1732779704862/Put/seqid=0 2024-11-28T07:41:45,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742112_1288 (size=12459) 2024-11-28T07:41:45,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742111_1287 (size=12459) 2024-11-28T07:41:45,350 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/2506cb48f791468d8571deb6002f1813 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/2506cb48f791468d8571deb6002f1813 2024-11-28T07:41:45,356 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 2506cb48f791468d8571deb6002f1813(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:45,356 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:45,356 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779705329; duration=0sec 2024-11-28T07:41:45,356 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:45,356 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:45,356 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:45,357 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:45,357 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:45,357 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,357 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3421029670244fd58bacf51d9ee0c33c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b0fade93e97449548cbaa1d0c98718be, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6d1e5217d73049a8b8dd56ea09ff353b] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=35.5 K 2024-11-28T07:41:45,357 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3421029670244fd58bacf51d9ee0c33c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732779702622 2024-11-28T07:41:45,358 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0fade93e97449548cbaa1d0c98718be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732779702672 2024-11-28T07:41:45,358 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d1e5217d73049a8b8dd56ea09ff353b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732779703802 2024-11-28T07:41:45,371 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:45,372 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/adbdea35d8a940008127c8b2e8960def is 50, key is test_row_0/C:col10/1732779704862/Put/seqid=0 2024-11-28T07:41:45,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-28T07:41:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:45,375 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:41:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:45,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742113_1289 (size=12459) 2024-11-28T07:41:45,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/cc2336cd2ccc4cbcbfc55b54c3bac02b is 50, key is test_row_0/A:col10/1732779704910/Put/seqid=0 2024-11-28T07:41:45,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742114_1290 (size=12151) 2024-11-28T07:41:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T07:41:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:45,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:45,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779765560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779765564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779765670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779765671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,751 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/542a129efad94ad8a325643bcd255443 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/542a129efad94ad8a325643bcd255443 2024-11-28T07:41:45,755 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into 542a129efad94ad8a325643bcd255443(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:45,755 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:45,755 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779705329; duration=0sec 2024-11-28T07:41:45,755 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:45,755 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:45,780 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/adbdea35d8a940008127c8b2e8960def as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/adbdea35d8a940008127c8b2e8960def 2024-11-28T07:41:45,785 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into adbdea35d8a940008127c8b2e8960def(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:45,785 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:45,785 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779705329; duration=0sec 2024-11-28T07:41:45,785 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/cc2336cd2ccc4cbcbfc55b54c3bac02b 2024-11-28T07:41:45,785 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:45,785 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:45,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/40838772b79b43e8a7800a85e1d7a4c1 is 50, key is test_row_0/B:col10/1732779704910/Put/seqid=0 2024-11-28T07:41:45,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742115_1291 (size=12151) 2024-11-28T07:41:45,796 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/40838772b79b43e8a7800a85e1d7a4c1 2024-11-28T07:41:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/f29c0ad0d0bb4116836155ace9775930 is 50, key is test_row_0/C:col10/1732779704910/Put/seqid=0 2024-11-28T07:41:45,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742116_1292 (size=12151) 2024-11-28T07:41:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779765880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779765880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779765970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779765970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:45,974 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:45,974 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:45,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T07:41:46,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779766190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:46,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779766190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:46,208 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/f29c0ad0d0bb4116836155ace9775930 2024-11-28T07:41:46,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/cc2336cd2ccc4cbcbfc55b54c3bac02b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc2336cd2ccc4cbcbfc55b54c3bac02b 2024-11-28T07:41:46,227 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc2336cd2ccc4cbcbfc55b54c3bac02b, entries=150, sequenceid=164, filesize=11.9 K 2024-11-28T07:41:46,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/40838772b79b43e8a7800a85e1d7a4c1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/40838772b79b43e8a7800a85e1d7a4c1 2024-11-28T07:41:46,232 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/40838772b79b43e8a7800a85e1d7a4c1, entries=150, sequenceid=164, filesize=11.9 K 2024-11-28T07:41:46,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/f29c0ad0d0bb4116836155ace9775930 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f29c0ad0d0bb4116836155ace9775930 2024-11-28T07:41:46,236 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f29c0ad0d0bb4116836155ace9775930, entries=150, sequenceid=164, filesize=11.9 K 2024-11-28T07:41:46,237 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ebed6cd25868ca618a5fe2887edb3192 in 862ms, sequenceid=164, compaction requested=false 2024-11-28T07:41:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:46,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-28T07:41:46,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-28T07:41:46,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-28T07:41:46,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3240 sec 2024-11-28T07:41:46,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3310 sec 2024-11-28T07:41:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:46,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:41:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:46,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:46,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/7197f9edd54347318acdd0e6e742cc2f is 50, key is test_row_0/A:col10/1732779705559/Put/seqid=0 2024-11-28T07:41:46,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742117_1293 (size=16931) 2024-11-28T07:41:46,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779766739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779766741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779766850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:46,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779766850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:46,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779766870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:46,873 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:47,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-28T07:41:47,060 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:47,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-28T07:41:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T07:41:47,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:47,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779767060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:47,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779767060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:47,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/7197f9edd54347318acdd0e6e742cc2f 2024-11-28T07:41:47,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/6d0ef4a335c34fb5aacf4afc9c489f94 is 50, key is test_row_0/B:col10/1732779705559/Put/seqid=0 2024-11-28T07:41:47,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742118_1294 (size=12151) 2024-11-28T07:41:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T07:41:47,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T07:41:47,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:47,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T07:41:47,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779767370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779767371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T07:41:47,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T07:41:47,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:47,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/6d0ef4a335c34fb5aacf4afc9c489f94 2024-11-28T07:41:47,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/693aaa4a156340ac9ce37fa6079ee5e1 is 50, key is test_row_0/C:col10/1732779705559/Put/seqid=0 2024-11-28T07:41:47,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742119_1295 (size=12151) 2024-11-28T07:41:47,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T07:41:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T07:41:47,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T07:41:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:47,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779767880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:47,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779767880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:47,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:47,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/693aaa4a156340ac9ce37fa6079ee5e1 2024-11-28T07:41:47,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/7197f9edd54347318acdd0e6e742cc2f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/7197f9edd54347318acdd0e6e742cc2f 2024-11-28T07:41:47,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/7197f9edd54347318acdd0e6e742cc2f, entries=250, sequenceid=181, filesize=16.5 K 2024-11-28T07:41:47,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/6d0ef4a335c34fb5aacf4afc9c489f94 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6d0ef4a335c34fb5aacf4afc9c489f94 2024-11-28T07:41:47,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6d0ef4a335c34fb5aacf4afc9c489f94, entries=150, sequenceid=181, filesize=11.9 K 2024-11-28T07:41:47,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/693aaa4a156340ac9ce37fa6079ee5e1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/693aaa4a156340ac9ce37fa6079ee5e1 2024-11-28T07:41:47,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/693aaa4a156340ac9ce37fa6079ee5e1, entries=150, sequenceid=181, filesize=11.9 K 2024-11-28T07:41:47,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ebed6cd25868ca618a5fe2887edb3192 in 1265ms, sequenceid=181, compaction requested=true 2024-11-28T07:41:47,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:47,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:47,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:47,968 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:47,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:47,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:47,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:47,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:47,969 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:47,970 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:47,970 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:47,970 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,970 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/2506cb48f791468d8571deb6002f1813, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc2336cd2ccc4cbcbfc55b54c3bac02b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/7197f9edd54347318acdd0e6e742cc2f] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=40.6 K 2024-11-28T07:41:47,970 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:47,970 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:47,970 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,970 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/542a129efad94ad8a325643bcd255443, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/40838772b79b43e8a7800a85e1d7a4c1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6d0ef4a335c34fb5aacf4afc9c489f94] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=35.9 K 2024-11-28T07:41:47,971 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2506cb48f791468d8571deb6002f1813, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732779703802 2024-11-28T07:41:47,971 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 542a129efad94ad8a325643bcd255443, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732779703802 2024-11-28T07:41:47,971 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc2336cd2ccc4cbcbfc55b54c3bac02b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732779704878 2024-11-28T07:41:47,971 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 40838772b79b43e8a7800a85e1d7a4c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732779704878 2024-11-28T07:41:47,971 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7197f9edd54347318acdd0e6e742cc2f, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732779705542 2024-11-28T07:41:47,972 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0ef4a335c34fb5aacf4afc9c489f94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732779705559 2024-11-28T07:41:47,980 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:47,981 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/29ebc801bd56446fa17d398e378712f3 is 50, key is test_row_0/A:col10/1732779705559/Put/seqid=0 2024-11-28T07:41:47,981 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:47,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-28T07:41:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:47,982 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:41:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:47,983 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/ee1b6f2592f94c0aa5bc834008e4bbfd is 50, key is test_row_0/B:col10/1732779705559/Put/seqid=0 2024-11-28T07:41:47,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742120_1296 (size=12561) 2024-11-28T07:41:47,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742121_1297 (size=12561) 2024-11-28T07:41:47,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/741ff2e92a184faa850e4a67bfbf8e05 is 50, key is test_row_0/A:col10/1732779706739/Put/seqid=0 2024-11-28T07:41:48,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742122_1298 (size=12151) 2024-11-28T07:41:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:48,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779768032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779768033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779768140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779768150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T07:41:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779768350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779768360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,393 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/29ebc801bd56446fa17d398e378712f3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/29ebc801bd56446fa17d398e378712f3 2024-11-28T07:41:48,397 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 29ebc801bd56446fa17d398e378712f3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:48,397 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:48,397 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779707968; duration=0sec 2024-11-28T07:41:48,398 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:48,398 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:48,398 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:48,399 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:48,399 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:48,399 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:48,399 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/adbdea35d8a940008127c8b2e8960def, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f29c0ad0d0bb4116836155ace9775930, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/693aaa4a156340ac9ce37fa6079ee5e1] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=35.9 K 2024-11-28T07:41:48,399 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting adbdea35d8a940008127c8b2e8960def, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732779703802 2024-11-28T07:41:48,399 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting f29c0ad0d0bb4116836155ace9775930, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732779704878 2024-11-28T07:41:48,400 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693aaa4a156340ac9ce37fa6079ee5e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732779705559 2024-11-28T07:41:48,402 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/ee1b6f2592f94c0aa5bc834008e4bbfd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/ee1b6f2592f94c0aa5bc834008e4bbfd 2024-11-28T07:41:48,403 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/741ff2e92a184faa850e4a67bfbf8e05 2024-11-28T07:41:48,407 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into ee1b6f2592f94c0aa5bc834008e4bbfd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:48,407 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:48,407 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779707968; duration=0sec 2024-11-28T07:41:48,407 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:48,407 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:48,409 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:48,410 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/83a206a114f742deac687c0082781ac0 is 50, key is test_row_0/C:col10/1732779705559/Put/seqid=0 2024-11-28T07:41:48,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/8715144e22014cf9b0acbc64d1cf6579 is 50, key is test_row_0/B:col10/1732779706739/Put/seqid=0 2024-11-28T07:41:48,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742123_1299 (size=12561) 2024-11-28T07:41:48,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742124_1300 (size=12151) 2024-11-28T07:41:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779768660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779768670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,815 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/8715144e22014cf9b0acbc64d1cf6579 2024-11-28T07:41:48,819 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/83a206a114f742deac687c0082781ac0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/83a206a114f742deac687c0082781ac0 2024-11-28T07:41:48,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/ced925490d64421c9cf8e6b76a36c803 is 50, key is test_row_0/C:col10/1732779706739/Put/seqid=0 2024-11-28T07:41:48,825 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into 83a206a114f742deac687c0082781ac0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:48,825 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:48,825 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779707969; duration=0sec 2024-11-28T07:41:48,825 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:48,825 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:48,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742125_1301 (size=12151) 2024-11-28T07:41:48,828 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/ced925490d64421c9cf8e6b76a36c803 2024-11-28T07:41:48,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/741ff2e92a184faa850e4a67bfbf8e05 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/741ff2e92a184faa850e4a67bfbf8e05 2024-11-28T07:41:48,835 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/741ff2e92a184faa850e4a67bfbf8e05, entries=150, sequenceid=198, filesize=11.9 K 2024-11-28T07:41:48,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/8715144e22014cf9b0acbc64d1cf6579 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/8715144e22014cf9b0acbc64d1cf6579 2024-11-28T07:41:48,840 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/8715144e22014cf9b0acbc64d1cf6579, entries=150, sequenceid=198, filesize=11.9 K 2024-11-28T07:41:48,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/ced925490d64421c9cf8e6b76a36c803 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ced925490d64421c9cf8e6b76a36c803 2024-11-28T07:41:48,847 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ced925490d64421c9cf8e6b76a36c803, entries=150, sequenceid=198, filesize=11.9 K 2024-11-28T07:41:48,848 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ebed6cd25868ca618a5fe2887edb3192 in 866ms, sequenceid=198, compaction requested=false 2024-11-28T07:41:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-28T07:41:48,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-28T07:41:48,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-28T07:41:48,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7860 sec 2024-11-28T07:41:48,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.7920 sec 2024-11-28T07:41:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:48,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:41:48,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:48,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:48,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:48,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:48,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:48,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:48,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/cc5bd13e6b9d433380cb25f04f463a63 is 50, key is test_row_0/A:col10/1732779708025/Put/seqid=0 2024-11-28T07:41:48,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742126_1302 (size=12151) 2024-11-28T07:41:48,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/cc5bd13e6b9d433380cb25f04f463a63 2024-11-28T07:41:48,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/82fb22c3d272488cb0d0e38fd92d6821 is 50, key is test_row_0/B:col10/1732779708025/Put/seqid=0 2024-11-28T07:41:48,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742127_1303 (size=12151) 2024-11-28T07:41:48,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/82fb22c3d272488cb0d0e38fd92d6821 2024-11-28T07:41:48,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/3e8a06fcdd184c739763f2f4cdf80619 is 50, key is test_row_0/C:col10/1732779708025/Put/seqid=0 2024-11-28T07:41:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779768930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742128_1304 (size=12151) 2024-11-28T07:41:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:48,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779768932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779769040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779769041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779769170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779769180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-28T07:41:49,210 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:49,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:49,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-28T07:41:49,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T07:41:49,213 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:49,214 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:49,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779769250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779769250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T07:41:49,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/3e8a06fcdd184c739763f2f4cdf80619 2024-11-28T07:41:49,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/cc5bd13e6b9d433380cb25f04f463a63 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc5bd13e6b9d433380cb25f04f463a63 2024-11-28T07:41:49,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc5bd13e6b9d433380cb25f04f463a63, entries=150, sequenceid=217, filesize=11.9 K 2024-11-28T07:41:49,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/82fb22c3d272488cb0d0e38fd92d6821 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/82fb22c3d272488cb0d0e38fd92d6821 2024-11-28T07:41:49,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/82fb22c3d272488cb0d0e38fd92d6821, entries=150, sequenceid=217, filesize=11.9 K 2024-11-28T07:41:49,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/3e8a06fcdd184c739763f2f4cdf80619 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3e8a06fcdd184c739763f2f4cdf80619 2024-11-28T07:41:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3e8a06fcdd184c739763f2f4cdf80619, entries=150, sequenceid=217, filesize=11.9 K 2024-11-28T07:41:49,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ebed6cd25868ca618a5fe2887edb3192 in 469ms, sequenceid=217, compaction requested=true 2024-11-28T07:41:49,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:49,362 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:49,362 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:49,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:49,363 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:49,363 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:49,363 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:49,363 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:49,363 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:49,363 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:49,363 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/ee1b6f2592f94c0aa5bc834008e4bbfd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/8715144e22014cf9b0acbc64d1cf6579, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/82fb22c3d272488cb0d0e38fd92d6821] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.0 K 2024-11-28T07:41:49,363 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/29ebc801bd56446fa17d398e378712f3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/741ff2e92a184faa850e4a67bfbf8e05, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc5bd13e6b9d433380cb25f04f463a63] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.0 K 2024-11-28T07:41:49,364 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1b6f2592f94c0aa5bc834008e4bbfd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732779705559 2024-11-28T07:41:49,364 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ebc801bd56446fa17d398e378712f3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732779705559 2024-11-28T07:41:49,364 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 8715144e22014cf9b0acbc64d1cf6579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732779706728 2024-11-28T07:41:49,364 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 741ff2e92a184faa850e4a67bfbf8e05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732779706728 2024-11-28T07:41:49,365 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 82fb22c3d272488cb0d0e38fd92d6821, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732779708025 2024-11-28T07:41:49,365 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5bd13e6b9d433380cb25f04f463a63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732779708025 2024-11-28T07:41:49,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-28T07:41:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:49,366 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:41:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:49,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:49,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:49,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/01f80c56164a4888bd43b07dc54c05a0 is 50, key is test_row_0/A:col10/1732779708931/Put/seqid=0 2024-11-28T07:41:49,395 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:49,395 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/fdc4c16ffbb04aa19b5bbfde4c27fcf3 is 50, key is test_row_0/B:col10/1732779708025/Put/seqid=0 2024-11-28T07:41:49,406 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:49,407 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4dc0d1ea456d4d789c0ac55566aa6cc5 is 50, key is test_row_0/A:col10/1732779708025/Put/seqid=0 2024-11-28T07:41:49,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742129_1305 (size=12151) 2024-11-28T07:41:49,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742130_1306 (size=12663) 2024-11-28T07:41:49,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742131_1307 (size=12663) 2024-11-28T07:41:49,452 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/fdc4c16ffbb04aa19b5bbfde4c27fcf3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/fdc4c16ffbb04aa19b5bbfde4c27fcf3 2024-11-28T07:41:49,465 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4dc0d1ea456d4d789c0ac55566aa6cc5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4dc0d1ea456d4d789c0ac55566aa6cc5 2024-11-28T07:41:49,475 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into fdc4c16ffbb04aa19b5bbfde4c27fcf3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:49,476 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:49,476 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779709362; duration=0sec 2024-11-28T07:41:49,476 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 4dc0d1ea456d4d789c0ac55566aa6cc5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:49,476 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:49,476 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:49,476 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:49,476 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779709362; duration=0sec 2024-11-28T07:41:49,476 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:49,476 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:49,476 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:49,480 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:49,480 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:49,481 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:49,481 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/83a206a114f742deac687c0082781ac0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ced925490d64421c9cf8e6b76a36c803, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3e8a06fcdd184c739763f2f4cdf80619] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.0 K 2024-11-28T07:41:49,482 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 83a206a114f742deac687c0082781ac0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732779705559 2024-11-28T07:41:49,482 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ced925490d64421c9cf8e6b76a36c803, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732779706728 2024-11-28T07:41:49,483 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e8a06fcdd184c739763f2f4cdf80619, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732779708025 2024-11-28T07:41:49,499 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:49,500 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/754c243913054e3b9bca5a010b9011ca is 50, key is test_row_0/C:col10/1732779708025/Put/seqid=0 2024-11-28T07:41:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T07:41:49,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742132_1308 (size=12663) 2024-11-28T07:41:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:49,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:49,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779769600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779769600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779769710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779769710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,834 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/01f80c56164a4888bd43b07dc54c05a0 2024-11-28T07:41:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T07:41:49,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/acd6fac5e05f46af8b32511b68ea83f9 is 50, key is test_row_0/B:col10/1732779708931/Put/seqid=0 2024-11-28T07:41:49,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742133_1309 (size=12151) 2024-11-28T07:41:49,874 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/acd6fac5e05f46af8b32511b68ea83f9 2024-11-28T07:41:49,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/04a5de6e5c6147b3a116892c495d5c39 is 50, key is test_row_0/C:col10/1732779708931/Put/seqid=0 2024-11-28T07:41:49,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742134_1310 (size=12151) 2024-11-28T07:41:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779769920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:49,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779769920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:49,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:49,926 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/04a5de6e5c6147b3a116892c495d5c39 2024-11-28T07:41:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/01f80c56164a4888bd43b07dc54c05a0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/01f80c56164a4888bd43b07dc54c05a0 2024-11-28T07:41:49,941 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/01f80c56164a4888bd43b07dc54c05a0, entries=150, sequenceid=233, filesize=11.9 K 2024-11-28T07:41:49,942 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/754c243913054e3b9bca5a010b9011ca as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754c243913054e3b9bca5a010b9011ca 2024-11-28T07:41:49,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/acd6fac5e05f46af8b32511b68ea83f9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/acd6fac5e05f46af8b32511b68ea83f9 2024-11-28T07:41:49,948 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into 754c243913054e3b9bca5a010b9011ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:49,949 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:49,949 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779709362; duration=0sec 2024-11-28T07:41:49,949 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:49,949 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:49,949 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/acd6fac5e05f46af8b32511b68ea83f9, entries=150, sequenceid=233, filesize=11.9 K 2024-11-28T07:41:49,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/04a5de6e5c6147b3a116892c495d5c39 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04a5de6e5c6147b3a116892c495d5c39 2024-11-28T07:41:49,955 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04a5de6e5c6147b3a116892c495d5c39, entries=150, sequenceid=233, filesize=11.9 K 2024-11-28T07:41:49,956 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ebed6cd25868ca618a5fe2887edb3192 in 590ms, sequenceid=233, compaction requested=false 2024-11-28T07:41:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-28T07:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-28T07:41:49,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-28T07:41:49,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 744 msec 2024-11-28T07:41:49,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 753 msec 2024-11-28T07:41:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:50,183 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:41:50,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:50,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:50,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:50,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:50,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:50,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:50,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/6996b1396abe4449a4b605c6e95ec73e is 50, key is test_row_0/A:col10/1732779709582/Put/seqid=0 2024-11-28T07:41:50,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742135_1311 (size=12151) 2024-11-28T07:41:50,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779770227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779770231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779770231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779770232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779770340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779770340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-28T07:41:50,351 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:50,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:50,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-28T07:41:50,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T07:41:50,353 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:50,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:50,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T07:41:50,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T07:41:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779770550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779770550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/6996b1396abe4449a4b605c6e95ec73e 2024-11-28T07:41:50,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/6305af867b134cae899fedc8c8f46cce is 50, key is test_row_0/B:col10/1732779709582/Put/seqid=0 2024-11-28T07:41:50,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742136_1312 (size=12151) 2024-11-28T07:41:50,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T07:41:50,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:50,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T07:41:50,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779770740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779770741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T07:41:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779770860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779770860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:50,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:50,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:50,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T07:41:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:50,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T07:41:51,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/6305af867b134cae899fedc8c8f46cce 2024-11-28T07:41:51,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/51e4a12e8a704a9bb708b96ba806beb7 is 50, key is test_row_0/C:col10/1732779709582/Put/seqid=0 2024-11-28T07:41:51,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742137_1313 (size=12151) 2024-11-28T07:41:51,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/51e4a12e8a704a9bb708b96ba806beb7 2024-11-28T07:41:51,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/6996b1396abe4449a4b605c6e95ec73e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6996b1396abe4449a4b605c6e95ec73e 2024-11-28T07:41:51,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6996b1396abe4449a4b605c6e95ec73e, entries=150, sequenceid=253, filesize=11.9 K 2024-11-28T07:41:51,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/6305af867b134cae899fedc8c8f46cce as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6305af867b134cae899fedc8c8f46cce 2024-11-28T07:41:51,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6305af867b134cae899fedc8c8f46cce, entries=150, sequenceid=253, filesize=11.9 K 2024-11-28T07:41:51,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/51e4a12e8a704a9bb708b96ba806beb7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/51e4a12e8a704a9bb708b96ba806beb7 2024-11-28T07:41:51,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/51e4a12e8a704a9bb708b96ba806beb7, entries=150, sequenceid=253, filesize=11.9 K 2024-11-28T07:41:51,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ebed6cd25868ca618a5fe2887edb3192 in 859ms, sequenceid=253, compaction requested=true 2024-11-28T07:41:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:51,043 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:51,043 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:51,049 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:51,049 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:51,049 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:51,049 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:51,049 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:51,049 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:51,049 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/fdc4c16ffbb04aa19b5bbfde4c27fcf3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/acd6fac5e05f46af8b32511b68ea83f9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6305af867b134cae899fedc8c8f46cce] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.1 K 2024-11-28T07:41:51,049 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4dc0d1ea456d4d789c0ac55566aa6cc5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/01f80c56164a4888bd43b07dc54c05a0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6996b1396abe4449a4b605c6e95ec73e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.1 K 2024-11-28T07:41:51,050 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc0d1ea456d4d789c0ac55566aa6cc5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732779708025 2024-11-28T07:41:51,050 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc4c16ffbb04aa19b5bbfde4c27fcf3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732779708025 2024-11-28T07:41:51,050 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01f80c56164a4888bd43b07dc54c05a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732779708914 2024-11-28T07:41:51,050 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting acd6fac5e05f46af8b32511b68ea83f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732779708914 2024-11-28T07:41:51,050 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 6305af867b134cae899fedc8c8f46cce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732779709573 2024-11-28T07:41:51,050 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6996b1396abe4449a4b605c6e95ec73e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732779709573 2024-11-28T07:41:51,060 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:51,061 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/7fd8f047092549cca7e6bd01c6845529 is 50, key is test_row_0/B:col10/1732779709582/Put/seqid=0 2024-11-28T07:41:51,066 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:51,067 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/9213d184c2f84e3cb30b602b9e8bc1cf is 50, key is test_row_0/A:col10/1732779709582/Put/seqid=0 2024-11-28T07:41:51,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742138_1314 (size=12765) 2024-11-28T07:41:51,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742139_1315 (size=12765) 2024-11-28T07:41:51,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-28T07:41:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:51,119 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:41:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:51,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:51,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/bc87e602725c43b5be13d65b65a94a0f is 50, key is test_row_0/A:col10/1732779710230/Put/seqid=0 2024-11-28T07:41:51,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742140_1316 (size=12301) 2024-11-28T07:41:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:51,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:51,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779771406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779771407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,475 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/7fd8f047092549cca7e6bd01c6845529 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7fd8f047092549cca7e6bd01c6845529 2024-11-28T07:41:51,480 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into 7fd8f047092549cca7e6bd01c6845529(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:51,480 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:51,480 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779711043; duration=0sec 2024-11-28T07:41:51,480 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:51,480 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:51,480 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:51,481 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:51,481 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:51,481 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:51,481 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754c243913054e3b9bca5a010b9011ca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04a5de6e5c6147b3a116892c495d5c39, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/51e4a12e8a704a9bb708b96ba806beb7] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.1 K 2024-11-28T07:41:51,482 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 754c243913054e3b9bca5a010b9011ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732779708025 2024-11-28T07:41:51,482 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 04a5de6e5c6147b3a116892c495d5c39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732779708914 2024-11-28T07:41:51,482 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e4a12e8a704a9bb708b96ba806beb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732779709573 2024-11-28T07:41:51,487 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/9213d184c2f84e3cb30b602b9e8bc1cf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/9213d184c2f84e3cb30b602b9e8bc1cf 2024-11-28T07:41:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T07:41:51,491 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:51,492 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 9213d184c2f84e3cb30b602b9e8bc1cf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:51,492 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:51,492 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779711043; duration=0sec 2024-11-28T07:41:51,492 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:51,492 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:51,492 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/bdf40cd1b3954e8c89bae80b5e7a191c is 50, key is test_row_0/C:col10/1732779709582/Put/seqid=0 2024-11-28T07:41:51,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742141_1317 (size=12765) 2024-11-28T07:41:51,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779771520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779771520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,539 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/bc87e602725c43b5be13d65b65a94a0f 2024-11-28T07:41:51,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/030728fe45cc4acc89e6701ae3f08a16 is 50, key is test_row_0/B:col10/1732779710230/Put/seqid=0 2024-11-28T07:41:51,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742142_1318 (size=12301) 2024-11-28T07:41:51,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779771730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779771730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779771760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779771760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:51,763 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,764 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:51,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:51,902 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/bdf40cd1b3954e8c89bae80b5e7a191c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/bdf40cd1b3954e8c89bae80b5e7a191c 2024-11-28T07:41:51,908 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into bdf40cd1b3954e8c89bae80b5e7a191c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:51,908 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:51,908 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779711043; duration=0sec 2024-11-28T07:41:51,908 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:51,908 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:51,955 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/030728fe45cc4acc89e6701ae3f08a16 2024-11-28T07:41:51,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/754a7441f55c46379b0a70650b9ed563 is 50, key is test_row_0/C:col10/1732779710230/Put/seqid=0 2024-11-28T07:41:51,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742143_1319 (size=12301) 2024-11-28T07:41:51,974 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/754a7441f55c46379b0a70650b9ed563 2024-11-28T07:41:51,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/bc87e602725c43b5be13d65b65a94a0f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bc87e602725c43b5be13d65b65a94a0f 2024-11-28T07:41:51,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bc87e602725c43b5be13d65b65a94a0f, entries=150, sequenceid=268, filesize=12.0 K 2024-11-28T07:41:51,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/030728fe45cc4acc89e6701ae3f08a16 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/030728fe45cc4acc89e6701ae3f08a16 2024-11-28T07:41:51,986 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/030728fe45cc4acc89e6701ae3f08a16, entries=150, sequenceid=268, filesize=12.0 K 2024-11-28T07:41:51,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/754a7441f55c46379b0a70650b9ed563 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754a7441f55c46379b0a70650b9ed563 2024-11-28T07:41:51,990 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754a7441f55c46379b0a70650b9ed563, entries=150, sequenceid=268, filesize=12.0 K 2024-11-28T07:41:51,991 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ebed6cd25868ca618a5fe2887edb3192 in 872ms, sequenceid=268, compaction requested=false 2024-11-28T07:41:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:51,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-28T07:41:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-28T07:41:51,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-28T07:41:51,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6380 sec 2024-11-28T07:41:51,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6440 sec 2024-11-28T07:41:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:52,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:41:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:52,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/66a3673064a14139958cc1d727c06883 is 50, key is test_row_0/A:col10/1732779711387/Put/seqid=0 2024-11-28T07:41:52,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742144_1320 (size=17181) 2024-11-28T07:41:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779772071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779772071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779772180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779772180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779772390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779772390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/66a3673064a14139958cc1d727c06883 2024-11-28T07:41:52,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/0226507da1e1477bb9aecb4f66eeeb62 is 50, key is test_row_0/B:col10/1732779711387/Put/seqid=0 2024-11-28T07:41:52,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742145_1321 (size=12301) 2024-11-28T07:41:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-28T07:41:52,500 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:52,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-28T07:41:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T07:41:52,502 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:52,503 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:52,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T07:41:52,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T07:41:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:52,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:52,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:52,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779772700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779772700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:52,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T07:41:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:52,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T07:41:52,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/0226507da1e1477bb9aecb4f66eeeb62 2024-11-28T07:41:52,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/a004804131754bbe80641fcfca138d69 is 50, key is test_row_0/C:col10/1732779711387/Put/seqid=0 2024-11-28T07:41:52,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742146_1322 (size=12301) 2024-11-28T07:41:52,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/a004804131754bbe80641fcfca138d69 2024-11-28T07:41:52,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/66a3673064a14139958cc1d727c06883 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/66a3673064a14139958cc1d727c06883 2024-11-28T07:41:52,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/66a3673064a14139958cc1d727c06883, entries=250, sequenceid=290, filesize=16.8 K 2024-11-28T07:41:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/0226507da1e1477bb9aecb4f66eeeb62 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/0226507da1e1477bb9aecb4f66eeeb62 2024-11-28T07:41:52,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/0226507da1e1477bb9aecb4f66eeeb62, entries=150, sequenceid=290, filesize=12.0 K 2024-11-28T07:41:52,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/a004804131754bbe80641fcfca138d69 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a004804131754bbe80641fcfca138d69 2024-11-28T07:41:52,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a004804131754bbe80641fcfca138d69, entries=150, sequenceid=290, filesize=12.0 K 2024-11-28T07:41:52,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for ebed6cd25868ca618a5fe2887edb3192 in 883ms, sequenceid=290, compaction requested=true 2024-11-28T07:41:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:52,927 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:52,927 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:52,928 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42247 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:52,928 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:52,928 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,928 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/9213d184c2f84e3cb30b602b9e8bc1cf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bc87e602725c43b5be13d65b65a94a0f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/66a3673064a14139958cc1d727c06883] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=41.3 K 2024-11-28T07:41:52,929 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:52,929 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:52,929 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9213d184c2f84e3cb30b602b9e8bc1cf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732779709573 2024-11-28T07:41:52,929 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,929 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7fd8f047092549cca7e6bd01c6845529, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/030728fe45cc4acc89e6701ae3f08a16, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/0226507da1e1477bb9aecb4f66eeeb62] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.5 K 2024-11-28T07:41:52,929 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc87e602725c43b5be13d65b65a94a0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732779710223 2024-11-28T07:41:52,929 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd8f047092549cca7e6bd01c6845529, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732779709573 2024-11-28T07:41:52,929 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a3673064a14139958cc1d727c06883, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779711387 2024-11-28T07:41:52,930 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 030728fe45cc4acc89e6701ae3f08a16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732779710223 2024-11-28T07:41:52,930 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0226507da1e1477bb9aecb4f66eeeb62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779711387 2024-11-28T07:41:52,938 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:52,938 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:52,938 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/84c109a17d954e70bfcf79045c7b9b77 is 50, key is test_row_0/A:col10/1732779711387/Put/seqid=0 2024-11-28T07:41:52,938 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/5848ea79acda497c976b865cd83b7114 is 50, key is test_row_0/B:col10/1732779711387/Put/seqid=0 2024-11-28T07:41:52,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742147_1323 (size=13017) 2024-11-28T07:41:52,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742148_1324 (size=13017) 2024-11-28T07:41:52,952 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/5848ea79acda497c976b865cd83b7114 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/5848ea79acda497c976b865cd83b7114 2024-11-28T07:41:52,958 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into 5848ea79acda497c976b865cd83b7114(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:52,958 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:52,958 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779712927; duration=0sec 2024-11-28T07:41:52,959 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:52,959 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:52,959 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:52,960 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:52,960 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:52,960 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,960 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/bdf40cd1b3954e8c89bae80b5e7a191c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754a7441f55c46379b0a70650b9ed563, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a004804131754bbe80641fcfca138d69] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.5 K 2024-11-28T07:41:52,960 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf40cd1b3954e8c89bae80b5e7a191c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732779709573 2024-11-28T07:41:52,960 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 754a7441f55c46379b0a70650b9ed563, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732779710223 2024-11-28T07:41:52,961 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a004804131754bbe80641fcfca138d69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779711387 2024-11-28T07:41:52,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-28T07:41:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:52,963 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:41:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:52,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:52,969 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:52,970 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/0c7418b6c61941808346e0dd72d94e9f is 50, key is test_row_0/C:col10/1732779711387/Put/seqid=0 2024-11-28T07:41:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/efd5792e06c14771a4430c2e32291d73 is 50, key is test_row_0/A:col10/1732779712069/Put/seqid=0 2024-11-28T07:41:52,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742149_1325 (size=13017) 2024-11-28T07:41:52,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742150_1326 (size=12301) 2024-11-28T07:41:52,982 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/0c7418b6c61941808346e0dd72d94e9f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0c7418b6c61941808346e0dd72d94e9f 2024-11-28T07:41:52,989 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into 0c7418b6c61941808346e0dd72d94e9f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:52,989 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:52,989 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779712927; duration=0sec 2024-11-28T07:41:52,989 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:52,989 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T07:41:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:53,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:53,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779773240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779773244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,348 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/84c109a17d954e70bfcf79045c7b9b77 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/84c109a17d954e70bfcf79045c7b9b77 2024-11-28T07:41:53,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779773350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779773350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,355 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 84c109a17d954e70bfcf79045c7b9b77(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:53,355 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:53,355 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779712927; duration=0sec 2024-11-28T07:41:53,355 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:53,355 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:53,377 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/efd5792e06c14771a4430c2e32291d73 2024-11-28T07:41:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/e7c8290e33da49439758713ebb600711 is 50, key is test_row_0/B:col10/1732779712069/Put/seqid=0 2024-11-28T07:41:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742151_1327 (size=12301) 2024-11-28T07:41:53,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779773561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779773561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T07:41:53,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779773770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,775 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,789 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/e7c8290e33da49439758713ebb600711 2024-11-28T07:41:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779773790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,792 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/ce01f82e07914d9cbe3f82e38d9c6668 is 50, key is test_row_0/C:col10/1732779712069/Put/seqid=0 2024-11-28T07:41:53,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742152_1328 (size=12301) 2024-11-28T07:41:53,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779773870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:53,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779773871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:54,219 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/ce01f82e07914d9cbe3f82e38d9c6668 2024-11-28T07:41:54,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/efd5792e06c14771a4430c2e32291d73 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/efd5792e06c14771a4430c2e32291d73 2024-11-28T07:41:54,227 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/efd5792e06c14771a4430c2e32291d73, entries=150, sequenceid=305, filesize=12.0 K 2024-11-28T07:41:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/e7c8290e33da49439758713ebb600711 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/e7c8290e33da49439758713ebb600711 2024-11-28T07:41:54,232 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/e7c8290e33da49439758713ebb600711, entries=150, sequenceid=305, filesize=12.0 K 2024-11-28T07:41:54,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/ce01f82e07914d9cbe3f82e38d9c6668 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ce01f82e07914d9cbe3f82e38d9c6668 2024-11-28T07:41:54,249 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ce01f82e07914d9cbe3f82e38d9c6668, entries=150, sequenceid=305, filesize=12.0 K 2024-11-28T07:41:54,249 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ebed6cd25868ca618a5fe2887edb3192 in 1286ms, sequenceid=305, compaction requested=false 2024-11-28T07:41:54,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:54,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:54,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-28T07:41:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-28T07:41:54,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-28T07:41:54,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7480 sec 2024-11-28T07:41:54,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7530 sec 2024-11-28T07:41:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:54,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:41:54,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:54,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:54,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:54,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:54,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:54,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:54,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/5c02971e48d34743a768c25eccbf7f5a is 50, key is test_row_0/A:col10/1732779713239/Put/seqid=0 2024-11-28T07:41:54,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742153_1329 (size=12301) 2024-11-28T07:41:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779774406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:54,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779774407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779774520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779774520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-28T07:41:54,651 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:54,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-28T07:41:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T07:41:54,653 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:54,653 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:54,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779774730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779774730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T07:41:54,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/5c02971e48d34743a768c25eccbf7f5a 2024-11-28T07:41:54,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/597556127f644e7f8c23240b7b663e2e is 50, key is test_row_0/B:col10/1732779713239/Put/seqid=0 2024-11-28T07:41:54,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T07:41:54,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:54,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:54,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742154_1330 (size=12301) 2024-11-28T07:41:54,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T07:41:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:54,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T07:41:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779775040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779775041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:55,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T07:41:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/597556127f644e7f8c23240b7b663e2e 2024-11-28T07:41:55,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/1ab201f0b51543d4baed663bce7c2737 is 50, key is test_row_0/C:col10/1732779713239/Put/seqid=0 2024-11-28T07:41:55,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742155_1331 (size=12301) 2024-11-28T07:41:55,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T07:41:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T07:41:55,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T07:41:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:55,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779775550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:55,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779775560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:55,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:55,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T07:41:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:55,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/1ab201f0b51543d4baed663bce7c2737 2024-11-28T07:41:55,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/5c02971e48d34743a768c25eccbf7f5a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/5c02971e48d34743a768c25eccbf7f5a 2024-11-28T07:41:55,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/5c02971e48d34743a768c25eccbf7f5a, entries=150, sequenceid=326, filesize=12.0 K 2024-11-28T07:41:55,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/597556127f644e7f8c23240b7b663e2e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/597556127f644e7f8c23240b7b663e2e 2024-11-28T07:41:55,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/597556127f644e7f8c23240b7b663e2e, entries=150, sequenceid=326, filesize=12.0 K 2024-11-28T07:41:55,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/1ab201f0b51543d4baed663bce7c2737 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1ab201f0b51543d4baed663bce7c2737 2024-11-28T07:41:55,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1ab201f0b51543d4baed663bce7c2737, entries=150, sequenceid=326, filesize=12.0 K 2024-11-28T07:41:55,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ebed6cd25868ca618a5fe2887edb3192 in 1263ms, sequenceid=326, compaction requested=true 2024-11-28T07:41:55,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:55,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:55,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:55,646 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:55,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:55,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:55,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:55,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-28T07:41:55,647 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:55,647 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:55,647 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,647 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:55,647 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/84c109a17d954e70bfcf79045c7b9b77, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/efd5792e06c14771a4430c2e32291d73, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/5c02971e48d34743a768c25eccbf7f5a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.7 K 2024-11-28T07:41:55,648 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c109a17d954e70bfcf79045c7b9b77, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779711387 2024-11-28T07:41:55,648 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd5792e06c14771a4430c2e32291d73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732779712047 2024-11-28T07:41:55,648 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:55,648 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:55,648 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,648 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/5848ea79acda497c976b865cd83b7114, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/e7c8290e33da49439758713ebb600711, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/597556127f644e7f8c23240b7b663e2e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.7 K 2024-11-28T07:41:55,649 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c02971e48d34743a768c25eccbf7f5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732779713238 2024-11-28T07:41:55,649 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5848ea79acda497c976b865cd83b7114, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779711387 2024-11-28T07:41:55,649 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c8290e33da49439758713ebb600711, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732779712047 2024-11-28T07:41:55,650 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 597556127f644e7f8c23240b7b663e2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732779713238 2024-11-28T07:41:55,656 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:55,657 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4d17d211df524f93b3707e6253c03241 is 50, key is test_row_0/A:col10/1732779713239/Put/seqid=0 2024-11-28T07:41:55,658 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:55,658 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/93c86b0886f3457da3ac52372f327ccb is 50, key is test_row_0/B:col10/1732779713239/Put/seqid=0 2024-11-28T07:41:55,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742156_1332 (size=13119) 2024-11-28T07:41:55,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742157_1333 (size=13119) 2024-11-28T07:41:55,667 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/4d17d211df524f93b3707e6253c03241 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4d17d211df524f93b3707e6253c03241 2024-11-28T07:41:55,671 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/93c86b0886f3457da3ac52372f327ccb as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/93c86b0886f3457da3ac52372f327ccb 2024-11-28T07:41:55,673 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 4d17d211df524f93b3707e6253c03241(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:55,673 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:55,673 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779715646; duration=0sec 2024-11-28T07:41:55,673 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:55,673 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:55,673 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:55,675 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:55,675 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:55,675 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,675 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0c7418b6c61941808346e0dd72d94e9f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ce01f82e07914d9cbe3f82e38d9c6668, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1ab201f0b51543d4baed663bce7c2737] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.7 K 2024-11-28T07:41:55,675 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c7418b6c61941808346e0dd72d94e9f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779711387 2024-11-28T07:41:55,676 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce01f82e07914d9cbe3f82e38d9c6668, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732779712047 2024-11-28T07:41:55,676 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ab201f0b51543d4baed663bce7c2737, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732779713238 2024-11-28T07:41:55,677 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into 93c86b0886f3457da3ac52372f327ccb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:55,677 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:55,677 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779715646; duration=0sec 2024-11-28T07:41:55,677 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:55,677 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:55,683 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:55,684 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/1b1e6008182b40e7a45f794f26689e72 is 50, key is test_row_0/C:col10/1732779713239/Put/seqid=0 2024-11-28T07:41:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742158_1334 (size=13119) 2024-11-28T07:41:55,696 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/1b1e6008182b40e7a45f794f26689e72 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1b1e6008182b40e7a45f794f26689e72 2024-11-28T07:41:55,703 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into 1b1e6008182b40e7a45f794f26689e72(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:55,703 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:55,703 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779715647; duration=0sec 2024-11-28T07:41:55,703 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:55,703 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:55,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-28T07:41:55,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:55,727 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:55,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:55,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/8e90177d5f9f49ccaf48cb9841c3570d is 50, key is test_row_0/A:col10/1732779714405/Put/seqid=0 2024-11-28T07:41:55,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742159_1335 (size=12301) 2024-11-28T07:41:55,738 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/8e90177d5f9f49ccaf48cb9841c3570d 2024-11-28T07:41:55,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/d05d205969a24b5f8b17c4e10f521e9e is 50, key is test_row_0/B:col10/1732779714405/Put/seqid=0 2024-11-28T07:41:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742160_1336 (size=12301) 2024-11-28T07:41:55,755 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/d05d205969a24b5f8b17c4e10f521e9e 2024-11-28T07:41:55,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b96bd7dccd5e4b63828331e7a21be9bb is 50, key is test_row_0/C:col10/1732779714405/Put/seqid=0 2024-11-28T07:41:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742161_1337 (size=12301) 2024-11-28T07:41:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T07:41:56,169 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b96bd7dccd5e4b63828331e7a21be9bb 2024-11-28T07:41:56,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/8e90177d5f9f49ccaf48cb9841c3570d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/8e90177d5f9f49ccaf48cb9841c3570d 2024-11-28T07:41:56,177 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/8e90177d5f9f49ccaf48cb9841c3570d, entries=150, sequenceid=342, filesize=12.0 K 2024-11-28T07:41:56,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/d05d205969a24b5f8b17c4e10f521e9e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d05d205969a24b5f8b17c4e10f521e9e 2024-11-28T07:41:56,182 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d05d205969a24b5f8b17c4e10f521e9e, entries=150, sequenceid=342, filesize=12.0 K 2024-11-28T07:41:56,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b96bd7dccd5e4b63828331e7a21be9bb as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b96bd7dccd5e4b63828331e7a21be9bb 2024-11-28T07:41:56,186 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b96bd7dccd5e4b63828331e7a21be9bb, entries=150, sequenceid=342, filesize=12.0 K 2024-11-28T07:41:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-28T07:41:56,187 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for ebed6cd25868ca618a5fe2887edb3192 in 460ms, sequenceid=342, compaction requested=false 2024-11-28T07:41:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:56,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:56,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-28T07:41:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-28T07:41:56,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-28T07:41:56,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5360 sec 2024-11-28T07:41:56,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.5410 sec 2024-11-28T07:41:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:56,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:41:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:56,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:56,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:56,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/bbeb703153ca4aca969c75a7eb310102 is 50, key is test_row_0/A:col10/1732779716565/Put/seqid=0 2024-11-28T07:41:56,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742162_1338 (size=17181) 2024-11-28T07:41:56,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/bbeb703153ca4aca969c75a7eb310102 2024-11-28T07:41:56,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/98f1613cab344637a950b4045fb24975 is 50, key is test_row_0/B:col10/1732779716565/Put/seqid=0 2024-11-28T07:41:56,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742163_1339 (size=12301) 2024-11-28T07:41:56,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/98f1613cab344637a950b4045fb24975 2024-11-28T07:41:56,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/35a0c169fa3e4ef8a463aef024a11254 is 50, key is test_row_0/C:col10/1732779716565/Put/seqid=0 2024-11-28T07:41:56,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742164_1340 (size=12301) 2024-11-28T07:41:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779716604, max timestamp=1732779716626Num uniques 859; 2024-11-28T07:41:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779716604, max timestamp=1732779716626Num uniques 859; 2024-11-28T07:41:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779716604, max timestamp=1732779716626Num uniques 859; 2024-11-28T07:41:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:41:56,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779776667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779776669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:56,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779776780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:56,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779776781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-28T07:41:56,800 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:56,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:41:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-28T07:41:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T07:41:56,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:41:56,803 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:41:56,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:41:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T07:41:56,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:56,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54004 deadline: 1732779776920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:56,923 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18280 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:56,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-28T07:41:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:56,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:41:56,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779776990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:56,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779776990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/35a0c169fa3e4ef8a463aef024a11254 2024-11-28T07:41:57,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/bbeb703153ca4aca969c75a7eb310102 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bbeb703153ca4aca969c75a7eb310102 2024-11-28T07:41:57,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bbeb703153ca4aca969c75a7eb310102, entries=250, sequenceid=352, filesize=16.8 K 2024-11-28T07:41:57,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/98f1613cab344637a950b4045fb24975 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/98f1613cab344637a950b4045fb24975 2024-11-28T07:41:57,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/98f1613cab344637a950b4045fb24975, entries=150, sequenceid=352, filesize=12.0 K 2024-11-28T07:41:57,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/35a0c169fa3e4ef8a463aef024a11254 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/35a0c169fa3e4ef8a463aef024a11254 2024-11-28T07:41:57,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/35a0c169fa3e4ef8a463aef024a11254, entries=150, sequenceid=352, filesize=12.0 K 2024-11-28T07:41:57,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ebed6cd25868ca618a5fe2887edb3192 in 463ms, sequenceid=352, compaction requested=true 2024-11-28T07:41:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:57,044 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:57,044 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:57,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:57,046 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:57,046 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:57,047 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:57,047 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/93c86b0886f3457da3ac52372f327ccb, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d05d205969a24b5f8b17c4e10f521e9e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/98f1613cab344637a950b4045fb24975] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.8 K 2024-11-28T07:41:57,047 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:57,047 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:57,047 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:57,047 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 93c86b0886f3457da3ac52372f327ccb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732779713238 2024-11-28T07:41:57,047 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4d17d211df524f93b3707e6253c03241, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/8e90177d5f9f49ccaf48cb9841c3570d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bbeb703153ca4aca969c75a7eb310102] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=41.6 K 2024-11-28T07:41:57,047 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d05d205969a24b5f8b17c4e10f521e9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732779714393 2024-11-28T07:41:57,048 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d17d211df524f93b3707e6253c03241, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732779713238 2024-11-28T07:41:57,048 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f1613cab344637a950b4045fb24975, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732779716565 2024-11-28T07:41:57,048 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e90177d5f9f49ccaf48cb9841c3570d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732779714393 2024-11-28T07:41:57,049 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbeb703153ca4aca969c75a7eb310102, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732779716565 2024-11-28T07:41:57,067 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:57,067 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:57,067 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/05bfae35dd4641eb8239bca3534027cc is 50, key is test_row_0/B:col10/1732779716565/Put/seqid=0 2024-11-28T07:41:57,067 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/211601d5e7124fa58bb66d1466b7f8e0 is 50, key is test_row_0/A:col10/1732779716565/Put/seqid=0 2024-11-28T07:41:57,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742166_1342 (size=13221) 2024-11-28T07:41:57,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742165_1341 (size=13221) 2024-11-28T07:41:57,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-28T07:41:57,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:57,110 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-28T07:41:57,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:57,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:57,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:57,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:57,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:57,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:41:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T07:41:57,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/a436d507fc004e939b0bd524d5640133 is 50, key is test_row_0/A:col10/1732779716667/Put/seqid=0 2024-11-28T07:41:57,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742167_1343 (size=12301) 2024-11-28T07:41:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:57,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. as already flushing 2024-11-28T07:41:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779777317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779777320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T07:41:57,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779777430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779777430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,491 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/05bfae35dd4641eb8239bca3534027cc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/05bfae35dd4641eb8239bca3534027cc 2024-11-28T07:41:57,496 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into 05bfae35dd4641eb8239bca3534027cc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:57,496 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:57,496 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779717044; duration=0sec 2024-11-28T07:41:57,497 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:57,497 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:41:57,497 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:57,498 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:57,498 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:41:57,498 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/211601d5e7124fa58bb66d1466b7f8e0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/211601d5e7124fa58bb66d1466b7f8e0 2024-11-28T07:41:57,498 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:57,498 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1b1e6008182b40e7a45f794f26689e72, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b96bd7dccd5e4b63828331e7a21be9bb, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/35a0c169fa3e4ef8a463aef024a11254] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=36.8 K 2024-11-28T07:41:57,499 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1e6008182b40e7a45f794f26689e72, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732779713238 2024-11-28T07:41:57,499 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting b96bd7dccd5e4b63828331e7a21be9bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732779714393 2024-11-28T07:41:57,499 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 35a0c169fa3e4ef8a463aef024a11254, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732779716565 2024-11-28T07:41:57,503 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into 211601d5e7124fa58bb66d1466b7f8e0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:57,503 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:57,503 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779717044; duration=0sec 2024-11-28T07:41:57,503 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:57,503 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:41:57,509 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:57,510 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/0d8cb63b157b45d3a69be585bbdbf999 is 50, key is test_row_0/C:col10/1732779716565/Put/seqid=0 2024-11-28T07:41:57,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742168_1344 (size=13221) 2024-11-28T07:41:57,539 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/a436d507fc004e939b0bd524d5640133 2024-11-28T07:41:57,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/85c4178e944941e4bc739c015eecb214 is 50, key is test_row_0/B:col10/1732779716667/Put/seqid=0 2024-11-28T07:41:57,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742169_1345 (size=12301) 2024-11-28T07:41:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779777640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779777640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53980 deadline: 1732779777810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,813 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53944 deadline: 1732779777820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,823 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in ebed6cd25868ca618a5fe2887edb3192 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:41:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,923 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/0d8cb63b157b45d3a69be585bbdbf999 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0d8cb63b157b45d3a69be585bbdbf999 2024-11-28T07:41:57,928 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into 0d8cb63b157b45d3a69be585bbdbf999(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:41:57,928 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:57,928 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779717044; duration=0sec 2024-11-28T07:41:57,928 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:57,928 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:41:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T07:41:57,950 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/85c4178e944941e4bc739c015eecb214 2024-11-28T07:41:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53938 deadline: 1732779777951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:41:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53976 deadline: 1732779777951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:41:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:41:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:41:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/808a7e1e81c24329b215d3c60bd8fe5d is 50, key is test_row_0/C:col10/1732779716667/Put/seqid=0 2024-11-28T07:41:57,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742170_1346 (size=12301) 2024-11-28T07:41:58,363 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/808a7e1e81c24329b215d3c60bd8fe5d 2024-11-28T07:41:58,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/a436d507fc004e939b0bd524d5640133 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a436d507fc004e939b0bd524d5640133 2024-11-28T07:41:58,375 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a436d507fc004e939b0bd524d5640133, entries=150, sequenceid=388, filesize=12.0 K 2024-11-28T07:41:58,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/85c4178e944941e4bc739c015eecb214 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/85c4178e944941e4bc739c015eecb214 2024-11-28T07:41:58,380 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/85c4178e944941e4bc739c015eecb214, entries=150, sequenceid=388, filesize=12.0 K 2024-11-28T07:41:58,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/808a7e1e81c24329b215d3c60bd8fe5d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/808a7e1e81c24329b215d3c60bd8fe5d 2024-11-28T07:41:58,384 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/808a7e1e81c24329b215d3c60bd8fe5d, entries=150, sequenceid=388, filesize=12.0 K 2024-11-28T07:41:58,385 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ebed6cd25868ca618a5fe2887edb3192 in 1276ms, sequenceid=388, compaction requested=false 2024-11-28T07:41:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:58,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-28T07:41:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-28T07:41:58,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-28T07:41:58,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-11-28T07:41:58,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.5880 sec 2024-11-28T07:41:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:41:58,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:41:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:41:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:41:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:41:58,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:41:58,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/6ad70de5964f4a0fb3045773ddd70ab4 is 50, key is test_row_1/A:col10/1732779718461/Put/seqid=0 2024-11-28T07:41:58,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742171_1347 (size=9857) 2024-11-28T07:41:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779718494, max timestamp=1732779718502Num uniques 859; 2024-11-28T07:41:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779718494, max timestamp=1732779718502Num uniques 859; 2024-11-28T07:41:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779718494, max timestamp=1732779718502Num uniques 859; 2024-11-28T07:41:58,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:41:58,519 INFO [Thread-1198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1198. 2024-11-28T07:41:58,519 DEBUG [Thread-1198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:58,519 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,519 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:58,520 INFO [Thread-1207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1207. 2024-11-28T07:41:58,520 DEBUG [Thread-1207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:58,520 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,520 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:58,521 INFO [Thread-1194 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1194. 2024-11-28T07:41:58,521 DEBUG [Thread-1194 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:58,521 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,521 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:58,523 INFO [Thread-1209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1209. 2024-11-28T07:41:58,523 DEBUG [Thread-1209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:58,523 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,523 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:58,523 INFO [Thread-1211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1211. 2024-11-28T07:41:58,523 DEBUG [Thread-1211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:58,523 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,523 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:58,524 INFO [Thread-1205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1205. 2024-11-28T07:41:58,524 DEBUG [Thread-1205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:58,524 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,524 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:58,525 INFO [Thread-1213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1213. 2024-11-28T07:41:58,525 DEBUG [Thread-1213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:41:58,525 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,525 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:41:58,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:41:58,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/6ad70de5964f4a0fb3045773ddd70ab4 2024-11-28T07:41:58,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/1bd22e8b93394d5b8bde6a2ff7539cb9 is 50, key is test_row_1/B:col10/1732779718461/Put/seqid=0 2024-11-28T07:41:58,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742172_1348 (size=9857) 2024-11-28T07:41:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-28T07:41:58,950 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:41:59,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/1bd22e8b93394d5b8bde6a2ff7539cb9 2024-11-28T07:41:59,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/d1b6da5399c9467ab6b6925b5c620968 is 50, key is test_row_1/C:col10/1732779718461/Put/seqid=0 2024-11-28T07:41:59,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742173_1349 (size=9857) 2024-11-28T07:41:59,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/d1b6da5399c9467ab6b6925b5c620968 2024-11-28T07:41:59,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/6ad70de5964f4a0fb3045773ddd70ab4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6ad70de5964f4a0fb3045773ddd70ab4 2024-11-28T07:41:59,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6ad70de5964f4a0fb3045773ddd70ab4, entries=100, sequenceid=402, filesize=9.6 K 2024-11-28T07:41:59,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/1bd22e8b93394d5b8bde6a2ff7539cb9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/1bd22e8b93394d5b8bde6a2ff7539cb9 2024-11-28T07:41:59,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/1bd22e8b93394d5b8bde6a2ff7539cb9, entries=100, sequenceid=402, filesize=9.6 K 2024-11-28T07:41:59,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/d1b6da5399c9467ab6b6925b5c620968 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/d1b6da5399c9467ab6b6925b5c620968 2024-11-28T07:41:59,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/d1b6da5399c9467ab6b6925b5c620968, entries=100, sequenceid=402, filesize=9.6 K 2024-11-28T07:41:59,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=73.80 KB/75570 for ebed6cd25868ca618a5fe2887edb3192 in 1243ms, sequenceid=402, compaction requested=true 2024-11-28T07:41:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:41:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:41:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:59,705 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:41:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:41:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ebed6cd25868ca618a5fe2887edb3192:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:41:59,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:41:59,705 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:41:59,706 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:59,706 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:41:59,706 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/A is initiating minor compaction (all files) 2024-11-28T07:41:59,706 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/A in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:59,706 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/B is initiating minor compaction (all files) 2024-11-28T07:41:59,706 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/211601d5e7124fa58bb66d1466b7f8e0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a436d507fc004e939b0bd524d5640133, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6ad70de5964f4a0fb3045773ddd70ab4] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=34.5 K 2024-11-28T07:41:59,706 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/B in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:41:59,706 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/05bfae35dd4641eb8239bca3534027cc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/85c4178e944941e4bc739c015eecb214, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/1bd22e8b93394d5b8bde6a2ff7539cb9] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=34.5 K 2024-11-28T07:41:59,706 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 211601d5e7124fa58bb66d1466b7f8e0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732779716565 2024-11-28T07:41:59,706 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 05bfae35dd4641eb8239bca3534027cc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732779716565 2024-11-28T07:41:59,707 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting a436d507fc004e939b0bd524d5640133, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732779716656 2024-11-28T07:41:59,707 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 85c4178e944941e4bc739c015eecb214, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732779716656 2024-11-28T07:41:59,707 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd22e8b93394d5b8bde6a2ff7539cb9, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732779717319 2024-11-28T07:41:59,707 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ad70de5964f4a0fb3045773ddd70ab4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732779717319 2024-11-28T07:41:59,714 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:59,714 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#A#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:41:59,714 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/f61b3268b16c4e818c066a54e199eb12 is 50, key is test_row_0/A:col10/1732779716667/Put/seqid=0 2024-11-28T07:41:59,714 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/92dab39179d14b72a90b4eca3b9fc501 is 50, key is test_row_0/B:col10/1732779716667/Put/seqid=0 2024-11-28T07:41:59,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742175_1351 (size=13323) 2024-11-28T07:41:59,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742174_1350 (size=13323) 2024-11-28T07:42:00,125 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/92dab39179d14b72a90b4eca3b9fc501 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/92dab39179d14b72a90b4eca3b9fc501 2024-11-28T07:42:00,125 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/f61b3268b16c4e818c066a54e199eb12 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f61b3268b16c4e818c066a54e199eb12 2024-11-28T07:42:00,129 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/B of ebed6cd25868ca618a5fe2887edb3192 into 92dab39179d14b72a90b4eca3b9fc501(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:00,129 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/A of ebed6cd25868ca618a5fe2887edb3192 into f61b3268b16c4e818c066a54e199eb12(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:00,129 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:42:00,129 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:42:00,129 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/B, priority=13, startTime=1732779719705; duration=0sec 2024-11-28T07:42:00,129 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/A, priority=13, startTime=1732779719705; duration=0sec 2024-11-28T07:42:00,130 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:00,130 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:B 2024-11-28T07:42:00,130 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:00,130 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:A 2024-11-28T07:42:00,130 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:00,130 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:00,130 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): ebed6cd25868ca618a5fe2887edb3192/C is initiating minor compaction (all files) 2024-11-28T07:42:00,130 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ebed6cd25868ca618a5fe2887edb3192/C in TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:42:00,130 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0d8cb63b157b45d3a69be585bbdbf999, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/808a7e1e81c24329b215d3c60bd8fe5d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/d1b6da5399c9467ab6b6925b5c620968] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp, totalSize=34.5 K 2024-11-28T07:42:00,131 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d8cb63b157b45d3a69be585bbdbf999, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732779716565 2024-11-28T07:42:00,131 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 808a7e1e81c24329b215d3c60bd8fe5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732779716656 2024-11-28T07:42:00,131 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b6da5399c9467ab6b6925b5c620968, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732779717319 2024-11-28T07:42:00,137 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ebed6cd25868ca618a5fe2887edb3192#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:00,137 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b4d23478174345b3ae5d2fcf09627f24 is 50, key is test_row_0/C:col10/1732779716667/Put/seqid=0 2024-11-28T07:42:00,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742176_1352 (size=13323) 2024-11-28T07:42:00,549 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/b4d23478174345b3ae5d2fcf09627f24 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b4d23478174345b3ae5d2fcf09627f24 2024-11-28T07:42:00,561 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ebed6cd25868ca618a5fe2887edb3192/C of ebed6cd25868ca618a5fe2887edb3192 into b4d23478174345b3ae5d2fcf09627f24(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:00,561 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:42:00,561 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., storeName=ebed6cd25868ca618a5fe2887edb3192/C, priority=13, startTime=1732779719705; duration=0sec 2024-11-28T07:42:00,561 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:00,561 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ebed6cd25868ca618a5fe2887edb3192:C 2024-11-28T07:42:03,593 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T07:42:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:42:07,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=80.51 KB heapSize=225.61 KB 2024-11-28T07:42:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:42:07,002 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-11-28T07:42:07,002 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:07,002 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:42:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:42:07,002 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:07,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:07,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/e9d247cdb5254514a165158e3170fb99 is 50, key is test_row_0/A:col10/1732779727001/Put/seqid=0 2024-11-28T07:42:07,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742177_1353 (size=12301) 2024-11-28T07:42:07,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/e9d247cdb5254514a165158e3170fb99 2024-11-28T07:42:07,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/db5255c00695471eabd45a709b4a4c8f is 50, key is test_row_0/B:col10/1732779727001/Put/seqid=0 2024-11-28T07:42:07,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742178_1354 (size=12301) 2024-11-28T07:42:07,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/db5255c00695471eabd45a709b4a4c8f 2024-11-28T07:42:07,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/c8811c6d5a7340d894d7746a8819a2ed is 50, key is test_row_0/C:col10/1732779727001/Put/seqid=0 2024-11-28T07:42:07,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742179_1355 (size=12301) 2024-11-28T07:42:07,912 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-11-28T07:42:07,912 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-11-28T07:42:07,912 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:07,912 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:07,912 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,912 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,912 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,912 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:07,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 4 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3540 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3513 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3479 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3546 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3509 2024-11-28T07:42:07,912 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T07:42:07,913 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:42:07,913 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:42:07,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:07,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:07,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T07:42:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:07,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T07:42:07,916 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779727916"}]},"ts":"1732779727916"} 2024-11-28T07:42:07,918 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T07:42:07,918 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T07:42:07,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:42:07,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, UNASSIGN}] 2024-11-28T07:42:07,920 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, UNASSIGN 2024-11-28T07:42:07,921 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ebed6cd25868ca618a5fe2887edb3192, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:07,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, UNASSIGN because future has completed 2024-11-28T07:42:07,922 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:42:07,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:42:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T07:42:08,074 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:42:08,074 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:42:08,074 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing ebed6cd25868ca618a5fe2887edb3192, disabling compactions & flushes 2024-11-28T07:42:08,074 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:42:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T07:42:08,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/c8811c6d5a7340d894d7746a8819a2ed 2024-11-28T07:42:08,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/e9d247cdb5254514a165158e3170fb99 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/e9d247cdb5254514a165158e3170fb99 2024-11-28T07:42:08,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/e9d247cdb5254514a165158e3170fb99, entries=150, sequenceid=434, filesize=12.0 K 2024-11-28T07:42:08,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/db5255c00695471eabd45a709b4a4c8f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/db5255c00695471eabd45a709b4a4c8f 2024-11-28T07:42:08,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/db5255c00695471eabd45a709b4a4c8f, entries=150, sequenceid=434, filesize=12.0 K 2024-11-28T07:42:08,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/c8811c6d5a7340d894d7746a8819a2ed as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/c8811c6d5a7340d894d7746a8819a2ed 2024-11-28T07:42:08,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/c8811c6d5a7340d894d7746a8819a2ed, entries=150, sequenceid=434, filesize=12.0 K 2024-11-28T07:42:08,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~225.56 KB/230976, currentSize=13.42 KB/13740 for ebed6cd25868ca618a5fe2887edb3192 in 1244ms, sequenceid=434, compaction requested=false 2024-11-28T07:42:08,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ebed6cd25868ca618a5fe2887edb3192: 2024-11-28T07:42:08,246 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:42:08,246 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:42:08,246 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. after waiting 0 ms 2024-11-28T07:42:08,246 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:42:08,246 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing ebed6cd25868ca618a5fe2887edb3192 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-28T07:42:08,247 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=A 2024-11-28T07:42:08,247 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:08,247 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=B 2024-11-28T07:42:08,247 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:08,247 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ebed6cd25868ca618a5fe2887edb3192, store=C 2024-11-28T07:42:08,247 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:08,250 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/c899c9e9a622472988ea318adfcd9813 is 50, key is test_row_0/A:col10/1732779727910/Put/seqid=0 2024-11-28T07:42:08,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742180_1356 (size=9857) 2024-11-28T07:42:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T07:42:08,654 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/c899c9e9a622472988ea318adfcd9813 2024-11-28T07:42:08,666 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/a777882175ef44ae974c54d808e3619e is 50, key is test_row_0/B:col10/1732779727910/Put/seqid=0 2024-11-28T07:42:08,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742181_1357 (size=9857) 2024-11-28T07:42:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T07:42:09,072 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/a777882175ef44ae974c54d808e3619e 2024-11-28T07:42:09,078 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/720ecfe4dc1748199ad4fa52e5bec0a1 is 50, key is test_row_0/C:col10/1732779727910/Put/seqid=0 2024-11-28T07:42:09,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742182_1358 (size=9857) 2024-11-28T07:42:09,482 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/720ecfe4dc1748199ad4fa52e5bec0a1 2024-11-28T07:42:09,487 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/A/c899c9e9a622472988ea318adfcd9813 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c899c9e9a622472988ea318adfcd9813 2024-11-28T07:42:09,495 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c899c9e9a622472988ea318adfcd9813, entries=100, sequenceid=439, filesize=9.6 K 2024-11-28T07:42:09,496 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/B/a777882175ef44ae974c54d808e3619e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/a777882175ef44ae974c54d808e3619e 2024-11-28T07:42:09,500 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/a777882175ef44ae974c54d808e3619e, entries=100, sequenceid=439, filesize=9.6 K 2024-11-28T07:42:09,505 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/.tmp/C/720ecfe4dc1748199ad4fa52e5bec0a1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/720ecfe4dc1748199ad4fa52e5bec0a1 2024-11-28T07:42:09,509 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/720ecfe4dc1748199ad4fa52e5bec0a1, entries=100, sequenceid=439, filesize=9.6 K 2024-11-28T07:42:09,510 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for ebed6cd25868ca618a5fe2887edb3192 in 1264ms, sequenceid=439, compaction requested=true 2024-11-28T07:42:09,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4e4833e209804bd28f8e5c29ca13eef3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/eab4da29af4b4d298f992c0d07a343c9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/ad9f2632ebde45cb9987c55883c06cb4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4988ef0ab23e47bebe233fe4b4f9b102, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c09fee080dd34ce7b85e020f62f33af1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f20207bbe19248139de7ca7e95c2da17, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/002c0fb24d494173970f16971a74457f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/503dc3c6306349f1a9c7ed1bd6aa90fc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a3aafd3218ac48f6b750814b60d0eaa2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/2506cb48f791468d8571deb6002f1813, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc2336cd2ccc4cbcbfc55b54c3bac02b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/7197f9edd54347318acdd0e6e742cc2f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/29ebc801bd56446fa17d398e378712f3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/741ff2e92a184faa850e4a67bfbf8e05, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4dc0d1ea456d4d789c0ac55566aa6cc5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc5bd13e6b9d433380cb25f04f463a63, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/01f80c56164a4888bd43b07dc54c05a0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/9213d184c2f84e3cb30b602b9e8bc1cf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6996b1396abe4449a4b605c6e95ec73e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bc87e602725c43b5be13d65b65a94a0f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/66a3673064a14139958cc1d727c06883, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/84c109a17d954e70bfcf79045c7b9b77, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/efd5792e06c14771a4430c2e32291d73, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4d17d211df524f93b3707e6253c03241, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/5c02971e48d34743a768c25eccbf7f5a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/8e90177d5f9f49ccaf48cb9841c3570d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bbeb703153ca4aca969c75a7eb310102, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/211601d5e7124fa58bb66d1466b7f8e0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a436d507fc004e939b0bd524d5640133, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6ad70de5964f4a0fb3045773ddd70ab4] to archive 2024-11-28T07:42:09,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:42:09,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4e4833e209804bd28f8e5c29ca13eef3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4e4833e209804bd28f8e5c29ca13eef3 2024-11-28T07:42:09,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/eab4da29af4b4d298f992c0d07a343c9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/eab4da29af4b4d298f992c0d07a343c9 2024-11-28T07:42:09,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/ad9f2632ebde45cb9987c55883c06cb4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/ad9f2632ebde45cb9987c55883c06cb4 2024-11-28T07:42:09,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4988ef0ab23e47bebe233fe4b4f9b102 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4988ef0ab23e47bebe233fe4b4f9b102 2024-11-28T07:42:09,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c09fee080dd34ce7b85e020f62f33af1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c09fee080dd34ce7b85e020f62f33af1 2024-11-28T07:42:09,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f20207bbe19248139de7ca7e95c2da17 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f20207bbe19248139de7ca7e95c2da17 2024-11-28T07:42:09,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/002c0fb24d494173970f16971a74457f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/002c0fb24d494173970f16971a74457f 2024-11-28T07:42:09,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/503dc3c6306349f1a9c7ed1bd6aa90fc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/503dc3c6306349f1a9c7ed1bd6aa90fc 2024-11-28T07:42:09,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a3aafd3218ac48f6b750814b60d0eaa2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a3aafd3218ac48f6b750814b60d0eaa2 2024-11-28T07:42:09,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/2506cb48f791468d8571deb6002f1813 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/2506cb48f791468d8571deb6002f1813 2024-11-28T07:42:09,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc2336cd2ccc4cbcbfc55b54c3bac02b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc2336cd2ccc4cbcbfc55b54c3bac02b 2024-11-28T07:42:09,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/7197f9edd54347318acdd0e6e742cc2f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/7197f9edd54347318acdd0e6e742cc2f 2024-11-28T07:42:09,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/29ebc801bd56446fa17d398e378712f3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/29ebc801bd56446fa17d398e378712f3 2024-11-28T07:42:09,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/741ff2e92a184faa850e4a67bfbf8e05 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/741ff2e92a184faa850e4a67bfbf8e05 2024-11-28T07:42:09,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4dc0d1ea456d4d789c0ac55566aa6cc5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4dc0d1ea456d4d789c0ac55566aa6cc5 2024-11-28T07:42:09,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc5bd13e6b9d433380cb25f04f463a63 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/cc5bd13e6b9d433380cb25f04f463a63 2024-11-28T07:42:09,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/01f80c56164a4888bd43b07dc54c05a0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/01f80c56164a4888bd43b07dc54c05a0 2024-11-28T07:42:09,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/9213d184c2f84e3cb30b602b9e8bc1cf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/9213d184c2f84e3cb30b602b9e8bc1cf 2024-11-28T07:42:09,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6996b1396abe4449a4b605c6e95ec73e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6996b1396abe4449a4b605c6e95ec73e 2024-11-28T07:42:09,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bc87e602725c43b5be13d65b65a94a0f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bc87e602725c43b5be13d65b65a94a0f 2024-11-28T07:42:09,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/66a3673064a14139958cc1d727c06883 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/66a3673064a14139958cc1d727c06883 2024-11-28T07:42:09,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/84c109a17d954e70bfcf79045c7b9b77 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/84c109a17d954e70bfcf79045c7b9b77 2024-11-28T07:42:09,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/efd5792e06c14771a4430c2e32291d73 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/efd5792e06c14771a4430c2e32291d73 2024-11-28T07:42:09,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4d17d211df524f93b3707e6253c03241 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/4d17d211df524f93b3707e6253c03241 2024-11-28T07:42:09,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/5c02971e48d34743a768c25eccbf7f5a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/5c02971e48d34743a768c25eccbf7f5a 2024-11-28T07:42:09,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/8e90177d5f9f49ccaf48cb9841c3570d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/8e90177d5f9f49ccaf48cb9841c3570d 2024-11-28T07:42:09,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bbeb703153ca4aca969c75a7eb310102 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/bbeb703153ca4aca969c75a7eb310102 2024-11-28T07:42:09,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/211601d5e7124fa58bb66d1466b7f8e0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/211601d5e7124fa58bb66d1466b7f8e0 2024-11-28T07:42:09,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a436d507fc004e939b0bd524d5640133 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/a436d507fc004e939b0bd524d5640133 2024-11-28T07:42:09,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6ad70de5964f4a0fb3045773ddd70ab4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/6ad70de5964f4a0fb3045773ddd70ab4 2024-11-28T07:42:09,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7771e1d42c2a415d8b85d66b331ff791, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/dffc6ee655b94bc79f2df590b8c3d9dc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/cc7e4bfc8ba040b78e5af00b616fd611, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/4539013062e84f8f8f0a04d3f2516589, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d2ee3dc53763488e803035ea7baba136, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/350b43cb63314543a44aba11d876d7f9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/363f5411a8f541378f5cb4ce754db70f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/315b7ed13fa1439185286f844dbe637f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/542a129efad94ad8a325643bcd255443, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/42921e884f1547eda7aa34f5621ef8bc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/40838772b79b43e8a7800a85e1d7a4c1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/ee1b6f2592f94c0aa5bc834008e4bbfd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6d0ef4a335c34fb5aacf4afc9c489f94, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/8715144e22014cf9b0acbc64d1cf6579, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/fdc4c16ffbb04aa19b5bbfde4c27fcf3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/82fb22c3d272488cb0d0e38fd92d6821, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/acd6fac5e05f46af8b32511b68ea83f9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7fd8f047092549cca7e6bd01c6845529, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6305af867b134cae899fedc8c8f46cce, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/030728fe45cc4acc89e6701ae3f08a16, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/5848ea79acda497c976b865cd83b7114, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/0226507da1e1477bb9aecb4f66eeeb62, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/e7c8290e33da49439758713ebb600711, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/93c86b0886f3457da3ac52372f327ccb, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/597556127f644e7f8c23240b7b663e2e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d05d205969a24b5f8b17c4e10f521e9e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/05bfae35dd4641eb8239bca3534027cc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/98f1613cab344637a950b4045fb24975, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/85c4178e944941e4bc739c015eecb214, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/1bd22e8b93394d5b8bde6a2ff7539cb9] to archive 2024-11-28T07:42:09,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:42:09,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7771e1d42c2a415d8b85d66b331ff791 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7771e1d42c2a415d8b85d66b331ff791 2024-11-28T07:42:09,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/dffc6ee655b94bc79f2df590b8c3d9dc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/dffc6ee655b94bc79f2df590b8c3d9dc 2024-11-28T07:42:09,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/cc7e4bfc8ba040b78e5af00b616fd611 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/cc7e4bfc8ba040b78e5af00b616fd611 2024-11-28T07:42:09,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/4539013062e84f8f8f0a04d3f2516589 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/4539013062e84f8f8f0a04d3f2516589 2024-11-28T07:42:09,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d2ee3dc53763488e803035ea7baba136 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d2ee3dc53763488e803035ea7baba136 2024-11-28T07:42:09,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/350b43cb63314543a44aba11d876d7f9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/350b43cb63314543a44aba11d876d7f9 2024-11-28T07:42:09,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/363f5411a8f541378f5cb4ce754db70f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/363f5411a8f541378f5cb4ce754db70f 2024-11-28T07:42:09,651 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/315b7ed13fa1439185286f844dbe637f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/315b7ed13fa1439185286f844dbe637f 2024-11-28T07:42:09,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/542a129efad94ad8a325643bcd255443 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/542a129efad94ad8a325643bcd255443 2024-11-28T07:42:09,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/42921e884f1547eda7aa34f5621ef8bc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/42921e884f1547eda7aa34f5621ef8bc 2024-11-28T07:42:09,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/40838772b79b43e8a7800a85e1d7a4c1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/40838772b79b43e8a7800a85e1d7a4c1 2024-11-28T07:42:09,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/ee1b6f2592f94c0aa5bc834008e4bbfd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/ee1b6f2592f94c0aa5bc834008e4bbfd 2024-11-28T07:42:09,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6d0ef4a335c34fb5aacf4afc9c489f94 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6d0ef4a335c34fb5aacf4afc9c489f94 2024-11-28T07:42:09,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/8715144e22014cf9b0acbc64d1cf6579 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/8715144e22014cf9b0acbc64d1cf6579 2024-11-28T07:42:09,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/fdc4c16ffbb04aa19b5bbfde4c27fcf3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/fdc4c16ffbb04aa19b5bbfde4c27fcf3 2024-11-28T07:42:09,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/82fb22c3d272488cb0d0e38fd92d6821 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/82fb22c3d272488cb0d0e38fd92d6821 2024-11-28T07:42:09,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/acd6fac5e05f46af8b32511b68ea83f9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/acd6fac5e05f46af8b32511b68ea83f9 2024-11-28T07:42:09,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7fd8f047092549cca7e6bd01c6845529 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/7fd8f047092549cca7e6bd01c6845529 2024-11-28T07:42:09,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6305af867b134cae899fedc8c8f46cce to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/6305af867b134cae899fedc8c8f46cce 2024-11-28T07:42:09,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/030728fe45cc4acc89e6701ae3f08a16 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/030728fe45cc4acc89e6701ae3f08a16 2024-11-28T07:42:09,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/5848ea79acda497c976b865cd83b7114 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/5848ea79acda497c976b865cd83b7114 2024-11-28T07:42:09,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/0226507da1e1477bb9aecb4f66eeeb62 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/0226507da1e1477bb9aecb4f66eeeb62 2024-11-28T07:42:09,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/e7c8290e33da49439758713ebb600711 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/e7c8290e33da49439758713ebb600711 2024-11-28T07:42:09,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/93c86b0886f3457da3ac52372f327ccb to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/93c86b0886f3457da3ac52372f327ccb 2024-11-28T07:42:09,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/597556127f644e7f8c23240b7b663e2e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/597556127f644e7f8c23240b7b663e2e 2024-11-28T07:42:09,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d05d205969a24b5f8b17c4e10f521e9e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/d05d205969a24b5f8b17c4e10f521e9e 2024-11-28T07:42:09,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/05bfae35dd4641eb8239bca3534027cc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/05bfae35dd4641eb8239bca3534027cc 2024-11-28T07:42:09,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/98f1613cab344637a950b4045fb24975 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/98f1613cab344637a950b4045fb24975 2024-11-28T07:42:09,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/85c4178e944941e4bc739c015eecb214 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/85c4178e944941e4bc739c015eecb214 2024-11-28T07:42:09,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/1bd22e8b93394d5b8bde6a2ff7539cb9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/1bd22e8b93394d5b8bde6a2ff7539cb9 2024-11-28T07:42:09,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6268725cc5834b48b1997dcc17a3097b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/4f1c169e95ce4d2ca5a80acf5b59b101, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04dff67ca8bf426c891f1935b28e0f8a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f70e3236cd5c4be286642b84acea8c8e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/29d229eb8f994aacb9fe3585eeb9255a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3421029670244fd58bacf51d9ee0c33c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a1d2ad410d0a411f9aa854db326ef64a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b0fade93e97449548cbaa1d0c98718be, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/adbdea35d8a940008127c8b2e8960def, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6d1e5217d73049a8b8dd56ea09ff353b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f29c0ad0d0bb4116836155ace9775930, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/83a206a114f742deac687c0082781ac0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/693aaa4a156340ac9ce37fa6079ee5e1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ced925490d64421c9cf8e6b76a36c803, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754c243913054e3b9bca5a010b9011ca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3e8a06fcdd184c739763f2f4cdf80619, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04a5de6e5c6147b3a116892c495d5c39, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/bdf40cd1b3954e8c89bae80b5e7a191c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/51e4a12e8a704a9bb708b96ba806beb7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754a7441f55c46379b0a70650b9ed563, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0c7418b6c61941808346e0dd72d94e9f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a004804131754bbe80641fcfca138d69, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ce01f82e07914d9cbe3f82e38d9c6668, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1b1e6008182b40e7a45f794f26689e72, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1ab201f0b51543d4baed663bce7c2737, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b96bd7dccd5e4b63828331e7a21be9bb, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0d8cb63b157b45d3a69be585bbdbf999, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/35a0c169fa3e4ef8a463aef024a11254, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/808a7e1e81c24329b215d3c60bd8fe5d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/d1b6da5399c9467ab6b6925b5c620968] to archive 2024-11-28T07:42:09,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:42:09,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6268725cc5834b48b1997dcc17a3097b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6268725cc5834b48b1997dcc17a3097b 2024-11-28T07:42:09,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/4f1c169e95ce4d2ca5a80acf5b59b101 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/4f1c169e95ce4d2ca5a80acf5b59b101 2024-11-28T07:42:09,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04dff67ca8bf426c891f1935b28e0f8a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04dff67ca8bf426c891f1935b28e0f8a 2024-11-28T07:42:09,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f70e3236cd5c4be286642b84acea8c8e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f70e3236cd5c4be286642b84acea8c8e 2024-11-28T07:42:09,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/29d229eb8f994aacb9fe3585eeb9255a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/29d229eb8f994aacb9fe3585eeb9255a 2024-11-28T07:42:09,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3421029670244fd58bacf51d9ee0c33c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3421029670244fd58bacf51d9ee0c33c 2024-11-28T07:42:09,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a1d2ad410d0a411f9aa854db326ef64a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a1d2ad410d0a411f9aa854db326ef64a 2024-11-28T07:42:09,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b0fade93e97449548cbaa1d0c98718be to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b0fade93e97449548cbaa1d0c98718be 2024-11-28T07:42:09,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/adbdea35d8a940008127c8b2e8960def to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/adbdea35d8a940008127c8b2e8960def 2024-11-28T07:42:09,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6d1e5217d73049a8b8dd56ea09ff353b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/6d1e5217d73049a8b8dd56ea09ff353b 2024-11-28T07:42:09,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f29c0ad0d0bb4116836155ace9775930 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/f29c0ad0d0bb4116836155ace9775930 2024-11-28T07:42:09,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/83a206a114f742deac687c0082781ac0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/83a206a114f742deac687c0082781ac0 2024-11-28T07:42:09,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/693aaa4a156340ac9ce37fa6079ee5e1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/693aaa4a156340ac9ce37fa6079ee5e1 2024-11-28T07:42:09,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ced925490d64421c9cf8e6b76a36c803 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ced925490d64421c9cf8e6b76a36c803 2024-11-28T07:42:09,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754c243913054e3b9bca5a010b9011ca to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754c243913054e3b9bca5a010b9011ca 2024-11-28T07:42:09,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3e8a06fcdd184c739763f2f4cdf80619 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/3e8a06fcdd184c739763f2f4cdf80619 2024-11-28T07:42:09,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04a5de6e5c6147b3a116892c495d5c39 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/04a5de6e5c6147b3a116892c495d5c39 2024-11-28T07:42:09,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/bdf40cd1b3954e8c89bae80b5e7a191c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/bdf40cd1b3954e8c89bae80b5e7a191c 2024-11-28T07:42:09,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/51e4a12e8a704a9bb708b96ba806beb7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/51e4a12e8a704a9bb708b96ba806beb7 2024-11-28T07:42:09,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754a7441f55c46379b0a70650b9ed563 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/754a7441f55c46379b0a70650b9ed563 2024-11-28T07:42:09,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0c7418b6c61941808346e0dd72d94e9f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0c7418b6c61941808346e0dd72d94e9f 2024-11-28T07:42:09,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a004804131754bbe80641fcfca138d69 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/a004804131754bbe80641fcfca138d69 2024-11-28T07:42:09,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ce01f82e07914d9cbe3f82e38d9c6668 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/ce01f82e07914d9cbe3f82e38d9c6668 2024-11-28T07:42:09,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1b1e6008182b40e7a45f794f26689e72 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1b1e6008182b40e7a45f794f26689e72 2024-11-28T07:42:09,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1ab201f0b51543d4baed663bce7c2737 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/1ab201f0b51543d4baed663bce7c2737 2024-11-28T07:42:09,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b96bd7dccd5e4b63828331e7a21be9bb to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b96bd7dccd5e4b63828331e7a21be9bb 2024-11-28T07:42:09,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0d8cb63b157b45d3a69be585bbdbf999 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/0d8cb63b157b45d3a69be585bbdbf999 2024-11-28T07:42:09,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/35a0c169fa3e4ef8a463aef024a11254 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/35a0c169fa3e4ef8a463aef024a11254 2024-11-28T07:42:09,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/808a7e1e81c24329b215d3c60bd8fe5d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/808a7e1e81c24329b215d3c60bd8fe5d 2024-11-28T07:42:09,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/d1b6da5399c9467ab6b6925b5c620968 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/d1b6da5399c9467ab6b6925b5c620968 2024-11-28T07:42:09,756 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/recovered.edits/442.seqid, newMaxSeqId=442, maxSeqId=1 2024-11-28T07:42:09,756 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. 2024-11-28T07:42:09,757 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for ebed6cd25868ca618a5fe2887edb3192: Waiting for close lock at 1732779728074Running coprocessor pre-close hooks at 1732779728074Disabling compacts and flushes for region at 1732779728074Disabling writes for close at 1732779728246 (+172 ms)Obtaining lock to block concurrent updates at 1732779728246Preparing flush snapshotting stores in ebed6cd25868ca618a5fe2887edb3192 at 1732779728246Finished memstore snapshotting TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732779728247 (+1 ms)Flushing stores of TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192. at 1732779728247Flushing ebed6cd25868ca618a5fe2887edb3192/A: creating writer at 1732779728247Flushing ebed6cd25868ca618a5fe2887edb3192/A: appending metadata at 1732779728249 (+2 ms)Flushing ebed6cd25868ca618a5fe2887edb3192/A: closing flushed file at 1732779728249Flushing ebed6cd25868ca618a5fe2887edb3192/B: creating writer at 1732779728658 (+409 ms)Flushing ebed6cd25868ca618a5fe2887edb3192/B: appending metadata at 1732779728665 (+7 ms)Flushing ebed6cd25868ca618a5fe2887edb3192/B: closing flushed file at 1732779728665Flushing ebed6cd25868ca618a5fe2887edb3192/C: creating writer at 1732779729075 (+410 ms)Flushing ebed6cd25868ca618a5fe2887edb3192/C: appending metadata at 1732779729077 (+2 ms)Flushing ebed6cd25868ca618a5fe2887edb3192/C: closing flushed file at 1732779729077Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78bff9d4: reopening flushed file at 1732779729486 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17bc6f87: reopening flushed file at 1732779729495 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@493e7882: reopening flushed file at 1732779729500 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for ebed6cd25868ca618a5fe2887edb3192 in 1264ms, sequenceid=439, compaction requested=true at 1732779729510 (+10 ms)Writing region close event to WAL at 1732779729752 (+242 ms)Running coprocessor post-close hooks at 1732779729756 (+4 ms)Closed at 1732779729756 2024-11-28T07:42:09,760 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=ebed6cd25868ca618a5fe2887edb3192, regionState=CLOSED 2024-11-28T07:42:09,761 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:42:09,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:42:09,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-28T07:42:09,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure ebed6cd25868ca618a5fe2887edb3192, server=f64cf8bdf19b,34437,1732779635619 in 1.8410 sec 2024-11-28T07:42:09,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-28T07:42:09,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ebed6cd25868ca618a5fe2887edb3192, UNASSIGN in 1.8450 sec 2024-11-28T07:42:09,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-28T07:42:09,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8500 sec 2024-11-28T07:42:09,772 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779729771"}]},"ts":"1732779729771"} 2024-11-28T07:42:09,774 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T07:42:09,774 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T07:42:09,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8620 sec 2024-11-28T07:42:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-28T07:42:10,061 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:10,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T07:42:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:10,063 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:10,064 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T07:42:10,067 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:42:10,069 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/recovered.edits] 2024-11-28T07:42:10,071 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c899c9e9a622472988ea318adfcd9813 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/c899c9e9a622472988ea318adfcd9813 2024-11-28T07:42:10,072 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/e9d247cdb5254514a165158e3170fb99 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/e9d247cdb5254514a165158e3170fb99 2024-11-28T07:42:10,073 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f61b3268b16c4e818c066a54e199eb12 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/A/f61b3268b16c4e818c066a54e199eb12 2024-11-28T07:42:10,078 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/92dab39179d14b72a90b4eca3b9fc501 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/92dab39179d14b72a90b4eca3b9fc501 2024-11-28T07:42:10,080 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/a777882175ef44ae974c54d808e3619e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/a777882175ef44ae974c54d808e3619e 2024-11-28T07:42:10,083 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/db5255c00695471eabd45a709b4a4c8f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/B/db5255c00695471eabd45a709b4a4c8f 2024-11-28T07:42:10,087 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/720ecfe4dc1748199ad4fa52e5bec0a1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/720ecfe4dc1748199ad4fa52e5bec0a1 2024-11-28T07:42:10,088 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b4d23478174345b3ae5d2fcf09627f24 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/b4d23478174345b3ae5d2fcf09627f24 2024-11-28T07:42:10,090 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/c8811c6d5a7340d894d7746a8819a2ed to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/C/c8811c6d5a7340d894d7746a8819a2ed 2024-11-28T07:42:10,094 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/recovered.edits/442.seqid to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192/recovered.edits/442.seqid 2024-11-28T07:42:10,094 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/ebed6cd25868ca618a5fe2887edb3192 2024-11-28T07:42:10,094 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T07:42:10,097 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:10,099 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T07:42:10,101 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T07:42:10,102 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:10,102 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T07:42:10,102 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732779730102"}]},"ts":"9223372036854775807"} 2024-11-28T07:42:10,104 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T07:42:10,104 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ebed6cd25868ca618a5fe2887edb3192, NAME => 'TestAcidGuarantees,,1732779697746.ebed6cd25868ca618a5fe2887edb3192.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T07:42:10,104 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T07:42:10,104 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732779730104"}]},"ts":"9223372036854775807"} 2024-11-28T07:42:10,106 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T07:42:10,107 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:10,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-28T07:42:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-28T07:42:10,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T07:42:10,171 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:10,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=421 (was 476), ProcessCount=11 (was 11), AvailableMemoryMB=6350 (was 7077) 2024-11-28T07:42:10,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=421, ProcessCount=11, AvailableMemoryMB=6349 2024-11-28T07:42:10,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:42:10,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:42:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:10,198 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:42:10,198 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:10,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-28T07:42:10,199 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:42:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T07:42:10,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742183_1359 (size=1061) 2024-11-28T07:42:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T07:42:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T07:42:10,606 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:42:10,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742184_1360 (size=53) 2024-11-28T07:42:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T07:42:11,011 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:42:11,011 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 963020bb226d03e8d0fb6ff8c63b3248, disabling compactions & flushes 2024-11-28T07:42:11,011 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,011 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,011 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. after waiting 0 ms 2024-11-28T07:42:11,011 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,011 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,011 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 963020bb226d03e8d0fb6ff8c63b3248: Waiting for close lock at 1732779731011Disabling compacts and flushes for region at 1732779731011Disabling writes for close at 1732779731011Writing region close event to WAL at 1732779731011Closed at 1732779731011 2024-11-28T07:42:11,012 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:42:11,013 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732779731012"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779731012"}]},"ts":"1732779731012"} 2024-11-28T07:42:11,014 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:42:11,015 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:42:11,015 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779731015"}]},"ts":"1732779731015"} 2024-11-28T07:42:11,016 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T07:42:11,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, ASSIGN}] 2024-11-28T07:42:11,017 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, ASSIGN 2024-11-28T07:42:11,018 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=false 2024-11-28T07:42:11,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:11,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, ASSIGN because future has completed 2024-11-28T07:42:11,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:42:11,325 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,325 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:42:11,325 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,325 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:42:11,325 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,325 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,327 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,328 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:11,328 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 963020bb226d03e8d0fb6ff8c63b3248 columnFamilyName A 2024-11-28T07:42:11,328 DEBUG [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:11,328 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(327): Store=963020bb226d03e8d0fb6ff8c63b3248/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:11,328 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,329 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:11,329 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 963020bb226d03e8d0fb6ff8c63b3248 columnFamilyName B 2024-11-28T07:42:11,329 DEBUG [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:11,330 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(327): Store=963020bb226d03e8d0fb6ff8c63b3248/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:11,330 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,330 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:11,331 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 963020bb226d03e8d0fb6ff8c63b3248 columnFamilyName C 2024-11-28T07:42:11,331 DEBUG [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:11,331 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(327): Store=963020bb226d03e8d0fb6ff8c63b3248/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:11,331 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,331 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,332 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,332 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,333 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,333 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,333 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:42:11,334 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,335 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:42:11,335 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 963020bb226d03e8d0fb6ff8c63b3248; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60756718, jitterRate=-0.09465435147285461}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:42:11,335 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:11,336 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 963020bb226d03e8d0fb6ff8c63b3248: Running coprocessor pre-open hook at 1732779731326Writing region info on filesystem at 1732779731326Initializing all the Stores at 1732779731326Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779731326Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779731326Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779731326Cleaning up temporary data from old regions at 1732779731333 (+7 ms)Running coprocessor post-open hooks at 1732779731335 (+2 ms)Region opened successfully at 1732779731336 (+1 ms) 2024-11-28T07:42:11,337 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., pid=95, masterSystemTime=1732779731322 2024-11-28T07:42:11,338 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,339 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:11,339 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T07:42:11,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:42:11,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-28T07:42:11,343 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 in 171 msec 2024-11-28T07:42:11,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-28T07:42:11,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, ASSIGN in 327 msec 2024-11-28T07:42:11,345 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:42:11,345 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779731345"}]},"ts":"1732779731345"} 2024-11-28T07:42:11,347 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T07:42:11,347 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:42:11,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-28T07:42:12,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-28T07:42:12,351 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:12,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:12,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:12,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:12,353 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c36da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:12,354 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56696, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@423b37b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:12,355 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:12,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:12,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:12,357 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:42:12,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,39063,1732779634677 2024-11-28T07:42:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36c67fda 2024-11-28T07:42:12,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:42:12,358 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:42:12,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:42:12,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:42:12,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:12,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742185_1361 (size=1097) 2024-11-28T07:42:12,771 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T07:42:12,771 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T07:42:12,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:42:12,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, REOPEN/MOVE}] 2024-11-28T07:42:12,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, REOPEN/MOVE 2024-11-28T07:42:12,776 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:12,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, REOPEN/MOVE because future has completed 2024-11-28T07:42:12,778 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:42:12,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:42:12,929 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:12,930 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:42:12,930 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 963020bb226d03e8d0fb6ff8c63b3248, disabling compactions & flushes 2024-11-28T07:42:12,930 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:12,930 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:12,930 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. after waiting 0 ms 2024-11-28T07:42:12,930 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:12,933 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:42:12,933 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:12,934 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 963020bb226d03e8d0fb6ff8c63b3248: Waiting for close lock at 1732779732930Running coprocessor pre-close hooks at 1732779732930Disabling compacts and flushes for region at 1732779732930Disabling writes for close at 1732779732930Writing region close event to WAL at 1732779732930Running coprocessor post-close hooks at 1732779732933 (+3 ms)Closed at 1732779732933 2024-11-28T07:42:12,934 WARN [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 963020bb226d03e8d0fb6ff8c63b3248 to self. 2024-11-28T07:42:12,935 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:12,936 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=CLOSED 2024-11-28T07:42:12,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:42:12,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-28T07:42:12,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 in 159 msec 2024-11-28T07:42:12,939 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, REOPEN/MOVE; state=CLOSED, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=true 2024-11-28T07:42:13,089 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, REOPEN/MOVE because future has completed 2024-11-28T07:42:13,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:42:13,245 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,246 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:42:13,246 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,246 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:42:13,246 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,246 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,247 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,248 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:13,248 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 963020bb226d03e8d0fb6ff8c63b3248 columnFamilyName A 2024-11-28T07:42:13,249 DEBUG [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:13,249 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(327): Store=963020bb226d03e8d0fb6ff8c63b3248/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:13,250 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,250 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:13,250 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 963020bb226d03e8d0fb6ff8c63b3248 columnFamilyName B 2024-11-28T07:42:13,250 DEBUG [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:13,251 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(327): Store=963020bb226d03e8d0fb6ff8c63b3248/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:13,251 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,251 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:13,251 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 963020bb226d03e8d0fb6ff8c63b3248 columnFamilyName C 2024-11-28T07:42:13,251 DEBUG [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:13,252 INFO [StoreOpener-963020bb226d03e8d0fb6ff8c63b3248-1 {}] regionserver.HStore(327): Store=963020bb226d03e8d0fb6ff8c63b3248/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:13,252 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,252 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,252 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,253 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,254 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,254 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,254 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:42:13,255 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,255 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 963020bb226d03e8d0fb6ff8c63b3248; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64709443, jitterRate=-0.03575415909290314}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:42:13,256 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,256 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 963020bb226d03e8d0fb6ff8c63b3248: Running coprocessor pre-open hook at 1732779733246Writing region info on filesystem at 1732779733246Initializing all the Stores at 1732779733247 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779733247Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779733247Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779733247Cleaning up temporary data from old regions at 1732779733254 (+7 ms)Running coprocessor post-open hooks at 1732779733256 (+2 ms)Region opened successfully at 1732779733256 2024-11-28T07:42:13,257 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., pid=100, masterSystemTime=1732779733242 2024-11-28T07:42:13,258 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,258 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,259 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=OPEN, openSeqNum=5, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:42:13,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-28T07:42:13,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 in 169 msec 2024-11-28T07:42:13,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-28T07:42:13,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, REOPEN/MOVE in 487 msec 2024-11-28T07:42:13,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-28T07:42:13,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-28T07:42:13,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-28T07:42:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-28T07:42:13,268 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:13,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,270 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8735c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,271 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e11f715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,273 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ee4ad9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,274 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52299357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b10f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,277 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12c84e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,279 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c866db5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,280 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61dc8b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,281 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b38e43b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,283 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2643d8d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,284 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,284 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@470234af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,285 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e055d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,287 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d074659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,288 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12cf886b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,289 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e60488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,290 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f983584, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f63200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,294 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ab870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:13,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:13,296 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53180c4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:13,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:13,297 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e15a16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:13,298 DEBUG [Thread-1635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,298 DEBUG [Thread-1639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,298 DEBUG [Thread-1648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,298 DEBUG [Thread-1654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,299 DEBUG [Thread-1643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,299 DEBUG [Thread-1652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:13,301 DEBUG [Thread-1650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,301 DEBUG [Thread-1646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-28T07:42:13,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,303 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:13,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T07:42:13,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,306 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:13,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:13,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,310 DEBUG [Thread-1637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:42:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:13,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:13,321 DEBUG [Thread-1641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:13,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:42:13,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281c994461093048be97e07e95199b6f1d_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779733316/Put/seqid=0 2024-11-28T07:42:13,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779733354, max timestamp=1732779733367Num uniques 859; 2024-11-28T07:42:13,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:42:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779733354, max timestamp=1732779733367Num uniques 859; 2024-11-28T07:42:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779733354, max timestamp=1732779733367Num uniques 859; 2024-11-28T07:42:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:42:13,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:42:13,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742186_1362 (size=12154) 2024-11-28T07:42:13,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:13,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281c994461093048be97e07e95199b6f1d_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281c994461093048be97e07e95199b6f1d_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:13,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c1297b2de5a14583985b576edf624706, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:13,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c1297b2de5a14583985b576edf624706 is 175, key is test_row_0/A:col10/1732779733316/Put/seqid=0 2024-11-28T07:42:13,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742187_1363 (size=30951) 2024-11-28T07:42:13,402 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c1297b2de5a14583985b576edf624706 2024-11-28T07:42:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T07:42:13,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779793406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779793408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779793410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779793412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779793414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,431 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/104e6cac4bf44a7a9e0e205b40d2c115 is 50, key is test_row_0/B:col10/1732779733316/Put/seqid=0 2024-11-28T07:42:13,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742188_1364 (size=9657) 2024-11-28T07:42:13,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T07:42:13,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:13,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779793520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779793530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779793531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779793531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779793540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T07:42:13,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:13,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T07:42:13,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779793730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779793750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779793750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779793750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T07:42:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:13,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779793760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:13,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/104e6cac4bf44a7a9e0e205b40d2c115 2024-11-28T07:42:13,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/355067e6072a40c49dcd5f29abea530a is 50, key is test_row_0/C:col10/1732779733316/Put/seqid=0 2024-11-28T07:42:13,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742189_1365 (size=9657) 2024-11-28T07:42:13,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T07:42:13,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:13,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:13,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T07:42:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779794050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779794070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779794070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779794070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T07:42:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:14,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:14,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:14,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779794081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T07:42:14,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:14,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:14,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:14,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:14,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:14,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/355067e6072a40c49dcd5f29abea530a 2024-11-28T07:42:14,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c1297b2de5a14583985b576edf624706 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c1297b2de5a14583985b576edf624706 2024-11-28T07:42:14,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c1297b2de5a14583985b576edf624706, entries=150, sequenceid=15, filesize=30.2 K 2024-11-28T07:42:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/104e6cac4bf44a7a9e0e205b40d2c115 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/104e6cac4bf44a7a9e0e205b40d2c115 2024-11-28T07:42:14,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/104e6cac4bf44a7a9e0e205b40d2c115, entries=100, sequenceid=15, filesize=9.4 K 2024-11-28T07:42:14,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/355067e6072a40c49dcd5f29abea530a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/355067e6072a40c49dcd5f29abea530a 2024-11-28T07:42:14,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/355067e6072a40c49dcd5f29abea530a, entries=100, sequenceid=15, filesize=9.4 K 2024-11-28T07:42:14,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 963020bb226d03e8d0fb6ff8c63b3248 in 980ms, sequenceid=15, compaction requested=false 2024-11-28T07:42:14,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:14,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-28T07:42:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:14,389 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T07:42:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:14,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:14,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128394efa711cb647edaae908b68e585706_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779733410/Put/seqid=0 2024-11-28T07:42:14,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742190_1366 (size=12154) 2024-11-28T07:42:14,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T07:42:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:14,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:14,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779794599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779794600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779794600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779794602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779794604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779794720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779794720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779794720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779794720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:14,806 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128394efa711cb647edaae908b68e585706_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128394efa711cb647edaae908b68e585706_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:14,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e82dd7a1706e4de382621a3a0efc55ff, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:14,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e82dd7a1706e4de382621a3a0efc55ff is 175, key is test_row_0/A:col10/1732779733410/Put/seqid=0 2024-11-28T07:42:14,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742191_1367 (size=30955) 2024-11-28T07:42:14,812 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e82dd7a1706e4de382621a3a0efc55ff 2024-11-28T07:42:14,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/055995612e86437e81a4ddfd2f5f1654 is 50, key is test_row_0/B:col10/1732779733410/Put/seqid=0 2024-11-28T07:42:14,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742192_1368 (size=12001) 2024-11-28T07:42:14,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779794940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779794940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779794940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779794940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,225 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/055995612e86437e81a4ddfd2f5f1654 2024-11-28T07:42:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/1116b649683142ceaf6936b58dc8bbd8 is 50, key is test_row_0/C:col10/1732779733410/Put/seqid=0 2024-11-28T07:42:15,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742193_1369 (size=12001) 2024-11-28T07:42:15,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779795261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779795261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779795261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779795261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,284 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:42:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T07:42:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779795620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,632 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,638 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/1116b649683142ceaf6936b58dc8bbd8 2024-11-28T07:42:15,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e82dd7a1706e4de382621a3a0efc55ff as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e82dd7a1706e4de382621a3a0efc55ff 2024-11-28T07:42:15,645 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e82dd7a1706e4de382621a3a0efc55ff, entries=150, sequenceid=50, filesize=30.2 K 2024-11-28T07:42:15,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/055995612e86437e81a4ddfd2f5f1654 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/055995612e86437e81a4ddfd2f5f1654 2024-11-28T07:42:15,649 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/055995612e86437e81a4ddfd2f5f1654, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T07:42:15,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/1116b649683142ceaf6936b58dc8bbd8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/1116b649683142ceaf6936b58dc8bbd8 2024-11-28T07:42:15,653 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/1116b649683142ceaf6936b58dc8bbd8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T07:42:15,654 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 963020bb226d03e8d0fb6ff8c63b3248 in 1265ms, sequenceid=50, compaction requested=false 2024-11-28T07:42:15,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:15,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:15,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-28T07:42:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-28T07:42:15,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-28T07:42:15,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3490 sec 2024-11-28T07:42:15,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3580 sec 2024-11-28T07:42:15,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:15,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:42:15,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:15,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:15,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:15,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:15,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:15,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:15,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289de4aee35a20472e845cda44f62cbf9a_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779735782/Put/seqid=0 2024-11-28T07:42:15,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742194_1370 (size=14594) 2024-11-28T07:42:15,807 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:15,811 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289de4aee35a20472e845cda44f62cbf9a_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289de4aee35a20472e845cda44f62cbf9a_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:15,812 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/746712c24efe43898d9ab276b64a482e, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:15,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/746712c24efe43898d9ab276b64a482e is 175, key is test_row_0/A:col10/1732779735782/Put/seqid=0 2024-11-28T07:42:15,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742195_1371 (size=39549) 2024-11-28T07:42:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779795823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779795828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779795830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779795830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779795940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779795940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779795950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779795950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779796161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779796161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779796161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779796161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/746712c24efe43898d9ab276b64a482e 2024-11-28T07:42:16,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/6dd3944b39ba4d709b653a0473bb52df is 50, key is test_row_0/B:col10/1732779735782/Put/seqid=0 2024-11-28T07:42:16,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742196_1372 (size=12001) 2024-11-28T07:42:16,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/6dd3944b39ba4d709b653a0473bb52df 2024-11-28T07:42:16,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/57bb40dc0bd042d981baaf0c7cfe9d83 is 50, key is test_row_0/C:col10/1732779735782/Put/seqid=0 2024-11-28T07:42:16,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742197_1373 (size=12001) 2024-11-28T07:42:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779796481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779796481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779796481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779796481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:16,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/57bb40dc0bd042d981baaf0c7cfe9d83 2024-11-28T07:42:16,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/746712c24efe43898d9ab276b64a482e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/746712c24efe43898d9ab276b64a482e 2024-11-28T07:42:16,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/746712c24efe43898d9ab276b64a482e, entries=200, sequenceid=62, filesize=38.6 K 2024-11-28T07:42:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/6dd3944b39ba4d709b653a0473bb52df as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/6dd3944b39ba4d709b653a0473bb52df 2024-11-28T07:42:16,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/6dd3944b39ba4d709b653a0473bb52df, entries=150, sequenceid=62, filesize=11.7 K 2024-11-28T07:42:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/57bb40dc0bd042d981baaf0c7cfe9d83 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/57bb40dc0bd042d981baaf0c7cfe9d83 2024-11-28T07:42:16,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/57bb40dc0bd042d981baaf0c7cfe9d83, entries=150, sequenceid=62, filesize=11.7 K 2024-11-28T07:42:16,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 963020bb226d03e8d0fb6ff8c63b3248 in 880ms, sequenceid=62, compaction requested=true 2024-11-28T07:42:16,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:16,665 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:16,665 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:16,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:16,666 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:16,666 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:16,666 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/B is initiating minor compaction (all files) 2024-11-28T07:42:16,666 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/A is initiating minor compaction (all files) 2024-11-28T07:42:16,666 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/B in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:16,666 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/A in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:16,666 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/104e6cac4bf44a7a9e0e205b40d2c115, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/055995612e86437e81a4ddfd2f5f1654, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/6dd3944b39ba4d709b653a0473bb52df] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=32.9 K 2024-11-28T07:42:16,666 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c1297b2de5a14583985b576edf624706, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e82dd7a1706e4de382621a3a0efc55ff, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/746712c24efe43898d9ab276b64a482e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=99.1 K 2024-11-28T07:42:16,666 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:16,666 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c1297b2de5a14583985b576edf624706, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e82dd7a1706e4de382621a3a0efc55ff, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/746712c24efe43898d9ab276b64a482e] 2024-11-28T07:42:16,667 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 104e6cac4bf44a7a9e0e205b40d2c115, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732779733316 2024-11-28T07:42:16,668 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 055995612e86437e81a4ddfd2f5f1654, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732779733406 2024-11-28T07:42:16,668 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1297b2de5a14583985b576edf624706, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732779733305 2024-11-28T07:42:16,669 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dd3944b39ba4d709b653a0473bb52df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779734599 2024-11-28T07:42:16,669 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82dd7a1706e4de382621a3a0efc55ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732779733406 2024-11-28T07:42:16,669 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 746712c24efe43898d9ab276b64a482e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779734599 2024-11-28T07:42:16,680 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#B#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:16,680 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/67becaedd9044430871155ad0ef7e4b8 is 50, key is test_row_0/B:col10/1732779735782/Put/seqid=0 2024-11-28T07:42:16,681 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:16,683 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411280560aad1a78b4bc498be67171f7aa510_963020bb226d03e8d0fb6ff8c63b3248 store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:16,684 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411280560aad1a78b4bc498be67171f7aa510_963020bb226d03e8d0fb6ff8c63b3248, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:16,684 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280560aad1a78b4bc498be67171f7aa510_963020bb226d03e8d0fb6ff8c63b3248 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:16,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742198_1374 (size=12104) 2024-11-28T07:42:16,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742199_1375 (size=4469) 2024-11-28T07:42:16,704 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/67becaedd9044430871155ad0ef7e4b8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/67becaedd9044430871155ad0ef7e4b8 2024-11-28T07:42:16,708 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/B of 963020bb226d03e8d0fb6ff8c63b3248 into 67becaedd9044430871155ad0ef7e4b8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:16,708 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:16,708 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/B, priority=13, startTime=1732779736665; duration=0sec 2024-11-28T07:42:16,708 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:16,708 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:B 2024-11-28T07:42:16,708 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:16,709 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:16,709 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/C is initiating minor compaction (all files) 2024-11-28T07:42:16,709 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/C in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:16,710 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/355067e6072a40c49dcd5f29abea530a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/1116b649683142ceaf6936b58dc8bbd8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/57bb40dc0bd042d981baaf0c7cfe9d83] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=32.9 K 2024-11-28T07:42:16,710 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 355067e6072a40c49dcd5f29abea530a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732779733316 2024-11-28T07:42:16,710 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1116b649683142ceaf6936b58dc8bbd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732779733406 2024-11-28T07:42:16,713 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 57bb40dc0bd042d981baaf0c7cfe9d83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779734599 2024-11-28T07:42:16,718 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#C#compaction#323 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:16,719 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/80dbb154abdd4b7e83853d56f042119d is 50, key is test_row_0/C:col10/1732779735782/Put/seqid=0 2024-11-28T07:42:16,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742200_1376 (size=12104) 2024-11-28T07:42:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:17,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-28T07:42:17,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779735797, max timestamp=1732779737007Num uniques 859; 2024-11-28T07:42:17,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:42:17,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779735797, max timestamp=1732779737007Num uniques 859; 2024-11-28T07:42:17,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:42:17,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:17,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:17,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:42:17,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283188d1386f074f0f8f1d242a8e246e9a_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779735821/Put/seqid=0 2024-11-28T07:42:17,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742201_1377 (size=12154) 2024-11-28T07:42:17,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779797060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779797061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779797066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779797068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,101 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:17,101 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/52e29b97fd92423c812d00be3ad11fd0 is 175, key is test_row_0/A:col10/1732779735782/Put/seqid=0 2024-11-28T07:42:17,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742202_1378 (size=31058) 2024-11-28T07:42:17,128 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/80dbb154abdd4b7e83853d56f042119d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/80dbb154abdd4b7e83853d56f042119d 2024-11-28T07:42:17,131 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/C of 963020bb226d03e8d0fb6ff8c63b3248 into 80dbb154abdd4b7e83853d56f042119d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:17,131 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:17,131 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/C, priority=13, startTime=1732779736665; duration=0sec 2024-11-28T07:42:17,132 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:17,132 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:C 2024-11-28T07:42:17,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779797170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779797180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779797191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779797191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779797390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779797390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779797410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779797410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:17,430 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411283188d1386f074f0f8f1d242a8e246e9a_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283188d1386f074f0f8f1d242a8e246e9a_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:17,430 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/2350fcb3451c482db076a0fda6141a11, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:17,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/2350fcb3451c482db076a0fda6141a11 is 175, key is test_row_0/A:col10/1732779735821/Put/seqid=0 2024-11-28T07:42:17,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742203_1379 (size=30955) 2024-11-28T07:42:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-28T07:42:17,491 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:17,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:17,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-28T07:42:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T07:42:17,493 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:17,494 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:17,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:17,510 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/52e29b97fd92423c812d00be3ad11fd0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/52e29b97fd92423c812d00be3ad11fd0 2024-11-28T07:42:17,514 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/A of 963020bb226d03e8d0fb6ff8c63b3248 into 52e29b97fd92423c812d00be3ad11fd0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:17,514 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:17,514 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/A, priority=13, startTime=1732779736664; duration=0sec 2024-11-28T07:42:17,514 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:17,514 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:A 2024-11-28T07:42:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T07:42:17,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:17,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779797660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,667 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779797700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779797710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779797730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779797730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:17,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:17,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T07:42:17,836 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/2350fcb3451c482db076a0fda6141a11 2024-11-28T07:42:17,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/be3a6af1751049ffa32469560723a299 is 50, key is test_row_0/B:col10/1732779735821/Put/seqid=0 2024-11-28T07:42:17,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742204_1380 (size=12001) 2024-11-28T07:42:17,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:17,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:18,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:18,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T07:42:18,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779798210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:18,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:18,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779798230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:18,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:18,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/be3a6af1751049ffa32469560723a299 2024-11-28T07:42:18,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/61be3b2c3a4047418fb2a09fec9941c7 is 50, key is test_row_0/C:col10/1732779735821/Put/seqid=0 2024-11-28T07:42:18,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742205_1381 (size=12001) 2024-11-28T07:42:18,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779798250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779798250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:18,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:18,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:18,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T07:42:18,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/61be3b2c3a4047418fb2a09fec9941c7 2024-11-28T07:42:18,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/2350fcb3451c482db076a0fda6141a11 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/2350fcb3451c482db076a0fda6141a11 2024-11-28T07:42:18,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/2350fcb3451c482db076a0fda6141a11, entries=150, sequenceid=85, filesize=30.2 K 2024-11-28T07:42:18,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/be3a6af1751049ffa32469560723a299 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/be3a6af1751049ffa32469560723a299 2024-11-28T07:42:18,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/be3a6af1751049ffa32469560723a299, entries=150, sequenceid=85, filesize=11.7 K 2024-11-28T07:42:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/61be3b2c3a4047418fb2a09fec9941c7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/61be3b2c3a4047418fb2a09fec9941c7 2024-11-28T07:42:18,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/61be3b2c3a4047418fb2a09fec9941c7, entries=150, sequenceid=85, filesize=11.7 K 2024-11-28T07:42:18,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 963020bb226d03e8d0fb6ff8c63b3248 in 1666ms, sequenceid=85, compaction requested=false 2024-11-28T07:42:18,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:18,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-28T07:42:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:18,719 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T07:42:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:18,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128672d2a89a8054bb3b9257cd5ba5c2655_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779737066/Put/seqid=0 2024-11-28T07:42:18,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742206_1382 (size=12154) 2024-11-28T07:42:19,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:19,137 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128672d2a89a8054bb3b9257cd5ba5c2655_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128672d2a89a8054bb3b9257cd5ba5c2655_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/f64189370df343be938fa52aee4f3a57, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:19,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/f64189370df343be938fa52aee4f3a57 is 175, key is test_row_0/A:col10/1732779737066/Put/seqid=0 2024-11-28T07:42:19,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742207_1383 (size=30955) 2024-11-28T07:42:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:19,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779799269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779799270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779799270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,275 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,276 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779799270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779799380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,387 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779799390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,551 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/f64189370df343be938fa52aee4f3a57 2024-11-28T07:42:19,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/4e3b7f67c49b4f6bbfff917b9ebd0707 is 50, key is test_row_0/B:col10/1732779737066/Put/seqid=0 2024-11-28T07:42:19,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742208_1384 (size=12001) 2024-11-28T07:42:19,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779799600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779799610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T07:42:19,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779799910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779799920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:19,966 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/4e3b7f67c49b4f6bbfff917b9ebd0707 2024-11-28T07:42:19,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/2c91575bffe24e9b9f5df63c1550733d is 50, key is test_row_0/C:col10/1732779737066/Put/seqid=0 2024-11-28T07:42:19,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742209_1385 (size=12001) 2024-11-28T07:42:19,978 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/2c91575bffe24e9b9f5df63c1550733d 2024-11-28T07:42:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/f64189370df343be938fa52aee4f3a57 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/f64189370df343be938fa52aee4f3a57 2024-11-28T07:42:19,986 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/f64189370df343be938fa52aee4f3a57, entries=150, sequenceid=107, filesize=30.2 K 2024-11-28T07:42:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/4e3b7f67c49b4f6bbfff917b9ebd0707 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/4e3b7f67c49b4f6bbfff917b9ebd0707 2024-11-28T07:42:19,992 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/4e3b7f67c49b4f6bbfff917b9ebd0707, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T07:42:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/2c91575bffe24e9b9f5df63c1550733d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2c91575bffe24e9b9f5df63c1550733d 2024-11-28T07:42:19,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2c91575bffe24e9b9f5df63c1550733d, entries=150, sequenceid=107, filesize=11.7 K 2024-11-28T07:42:19,996 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 963020bb226d03e8d0fb6ff8c63b3248 in 1277ms, sequenceid=107, compaction requested=true 2024-11-28T07:42:19,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:19,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:19,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-28T07:42:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-28T07:42:19,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-28T07:42:19,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5030 sec 2024-11-28T07:42:20,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.5090 sec 2024-11-28T07:42:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:20,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:42:20,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:20,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:20,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:20,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:20,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:20,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:20,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f0c21e96e0949bcae5e4518e64f0282_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779739267/Put/seqid=0 2024-11-28T07:42:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742210_1386 (size=14594) 2024-11-28T07:42:20,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779800580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779800583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:20,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779800700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779800710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:20,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:20,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282f0c21e96e0949bcae5e4518e64f0282_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f0c21e96e0949bcae5e4518e64f0282_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:20,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/9084f1955f2e4891a3834d46cc6f452d, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:20,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/9084f1955f2e4891a3834d46cc6f452d is 175, key is test_row_0/A:col10/1732779739267/Put/seqid=0 2024-11-28T07:42:20,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742211_1387 (size=39549) 2024-11-28T07:42:20,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779800910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779800930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779801230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779801250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/9084f1955f2e4891a3834d46cc6f452d 2024-11-28T07:42:21,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/bac94b9a8e95452fae7e9a499f7159ba is 50, key is test_row_0/B:col10/1732779739267/Put/seqid=0 2024-11-28T07:42:21,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742212_1388 (size=12001) 2024-11-28T07:42:21,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779801290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:21,295 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779801300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:21,309 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-28T07:42:21,661 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:21,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-28T07:42:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T07:42:21,663 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:21,663 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:21,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:21,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/bac94b9a8e95452fae7e9a499f7159ba 2024-11-28T07:42:21,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779801680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:21,694 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/db6ea553b0b244cb8a342dce79da6be3 is 50, key is test_row_0/C:col10/1732779739267/Put/seqid=0 2024-11-28T07:42:21,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742213_1389 (size=12001) 2024-11-28T07:42:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779801740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T07:42:21,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779801770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:21,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T07:42:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:21,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:21,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T07:42:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:21,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T07:42:22,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/db6ea553b0b244cb8a342dce79da6be3 2024-11-28T07:42:22,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/9084f1955f2e4891a3834d46cc6f452d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9084f1955f2e4891a3834d46cc6f452d 2024-11-28T07:42:22,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9084f1955f2e4891a3834d46cc6f452d, entries=200, sequenceid=120, filesize=38.6 K 2024-11-28T07:42:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/bac94b9a8e95452fae7e9a499f7159ba as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/bac94b9a8e95452fae7e9a499f7159ba 2024-11-28T07:42:22,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/bac94b9a8e95452fae7e9a499f7159ba, entries=150, sequenceid=120, filesize=11.7 K 2024-11-28T07:42:22,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/db6ea553b0b244cb8a342dce79da6be3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/db6ea553b0b244cb8a342dce79da6be3 2024-11-28T07:42:22,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/db6ea553b0b244cb8a342dce79da6be3, entries=150, sequenceid=120, filesize=11.7 K 2024-11-28T07:42:22,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 963020bb226d03e8d0fb6ff8c63b3248 in 1683ms, sequenceid=120, compaction requested=true 2024-11-28T07:42:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:22,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:22,118 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:42:22,118 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:42:22,119 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:42:22,120 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/A is initiating minor compaction (all files) 2024-11-28T07:42:22,120 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/A in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:22,120 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/52e29b97fd92423c812d00be3ad11fd0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/2350fcb3451c482db076a0fda6141a11, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/f64189370df343be938fa52aee4f3a57, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9084f1955f2e4891a3834d46cc6f452d] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=129.4 K 2024-11-28T07:42:22,120 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:22,120 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/52e29b97fd92423c812d00be3ad11fd0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/2350fcb3451c482db076a0fda6141a11, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/f64189370df343be938fa52aee4f3a57, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9084f1955f2e4891a3834d46cc6f452d] 2024-11-28T07:42:22,120 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:42:22,120 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/B is initiating minor compaction (all files) 2024-11-28T07:42:22,120 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/B in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:22,120 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/67becaedd9044430871155ad0ef7e4b8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/be3a6af1751049ffa32469560723a299, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/4e3b7f67c49b4f6bbfff917b9ebd0707, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/bac94b9a8e95452fae7e9a499f7159ba] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=47.0 K 2024-11-28T07:42:22,120 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e29b97fd92423c812d00be3ad11fd0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779734599 2024-11-28T07:42:22,121 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 67becaedd9044430871155ad0ef7e4b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779734599 2024-11-28T07:42:22,121 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2350fcb3451c482db076a0fda6141a11, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732779735821 2024-11-28T07:42:22,121 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting be3a6af1751049ffa32469560723a299, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732779735821 2024-11-28T07:42:22,122 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e3b7f67c49b4f6bbfff917b9ebd0707, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732779737046 2024-11-28T07:42:22,122 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting f64189370df343be938fa52aee4f3a57, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732779737046 2024-11-28T07:42:22,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-28T07:42:22,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:22,122 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting bac94b9a8e95452fae7e9a499f7159ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732779739267 2024-11-28T07:42:22,122 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9084f1955f2e4891a3834d46cc6f452d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732779739257 2024-11-28T07:42:22,122 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:42:22,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:22,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:22,129 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:22,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e935ff2226614368a2e5559631514c3b_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779740579/Put/seqid=0 2024-11-28T07:42:22,132 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#B#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:22,132 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/e8bc92cfccf64ab49b2da765ea6b8113 is 50, key is test_row_0/B:col10/1732779739267/Put/seqid=0 2024-11-28T07:42:22,134 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128c5c3f99ea509474faa0aa94610fd65d8_963020bb226d03e8d0fb6ff8c63b3248 store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:22,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742214_1390 (size=12304) 2024-11-28T07:42:22,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:22,138 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128c5c3f99ea509474faa0aa94610fd65d8_963020bb226d03e8d0fb6ff8c63b3248, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:22,138 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c5c3f99ea509474faa0aa94610fd65d8_963020bb226d03e8d0fb6ff8c63b3248 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:22,146 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e935ff2226614368a2e5559631514c3b_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e935ff2226614368a2e5559631514c3b_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742215_1391 (size=12241) 2024-11-28T07:42:22,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/d302e66b15e84f239c026cd206030a1f, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/d302e66b15e84f239c026cd206030a1f is 175, key is test_row_0/A:col10/1732779740579/Put/seqid=0 2024-11-28T07:42:22,154 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/e8bc92cfccf64ab49b2da765ea6b8113 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e8bc92cfccf64ab49b2da765ea6b8113 2024-11-28T07:42:22,159 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/B of 963020bb226d03e8d0fb6ff8c63b3248 into e8bc92cfccf64ab49b2da765ea6b8113(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:22,159 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:22,159 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/B, priority=12, startTime=1732779742118; duration=0sec 2024-11-28T07:42:22,159 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:22,159 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:B 2024-11-28T07:42:22,159 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:42:22,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742216_1392 (size=4469) 2024-11-28T07:42:22,161 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:42:22,161 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#A#compaction#335 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:22,161 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/C is initiating minor compaction (all files) 2024-11-28T07:42:22,161 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/C in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:22,161 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/80dbb154abdd4b7e83853d56f042119d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/61be3b2c3a4047418fb2a09fec9941c7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2c91575bffe24e9b9f5df63c1550733d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/db6ea553b0b244cb8a342dce79da6be3] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=47.0 K 2024-11-28T07:42:22,161 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 80dbb154abdd4b7e83853d56f042119d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732779734599 2024-11-28T07:42:22,161 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/792996f0098f4d67bab1424880d62628 is 175, key is test_row_0/A:col10/1732779739267/Put/seqid=0 2024-11-28T07:42:22,162 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 61be3b2c3a4047418fb2a09fec9941c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732779735821 2024-11-28T07:42:22,162 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c91575bffe24e9b9f5df63c1550733d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732779737046 2024-11-28T07:42:22,162 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting db6ea553b0b244cb8a342dce79da6be3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732779739267 2024-11-28T07:42:22,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742218_1394 (size=31195) 2024-11-28T07:42:22,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742217_1393 (size=31105) 2024-11-28T07:42:22,175 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:22,175 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/d1e72e5cc9394faa8f72180b3484e589 is 50, key is test_row_0/C:col10/1732779739267/Put/seqid=0 2024-11-28T07:42:22,178 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/792996f0098f4d67bab1424880d62628 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/792996f0098f4d67bab1424880d62628 2024-11-28T07:42:22,183 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/A of 963020bb226d03e8d0fb6ff8c63b3248 into 792996f0098f4d67bab1424880d62628(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:22,183 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:22,183 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/A, priority=12, startTime=1732779742118; duration=0sec 2024-11-28T07:42:22,183 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:22,183 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:A 2024-11-28T07:42:22,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742219_1395 (size=12241) 2024-11-28T07:42:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T07:42:22,575 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/d302e66b15e84f239c026cd206030a1f 2024-11-28T07:42:22,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/b0bd1b88eefa4c9589c02684ab1fa0a3 is 50, key is test_row_0/B:col10/1732779740579/Put/seqid=0 2024-11-28T07:42:22,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742220_1396 (size=12151) 2024-11-28T07:42:22,591 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/d1e72e5cc9394faa8f72180b3484e589 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/d1e72e5cc9394faa8f72180b3484e589 2024-11-28T07:42:22,595 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/C of 963020bb226d03e8d0fb6ff8c63b3248 into d1e72e5cc9394faa8f72180b3484e589(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:22,595 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:22,595 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/C, priority=12, startTime=1732779742118; duration=0sec 2024-11-28T07:42:22,595 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:22,595 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:C 2024-11-28T07:42:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:22,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T07:42:22,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779802807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779802809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:22,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779802920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779802920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:22,986 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/b0bd1b88eefa4c9589c02684ab1fa0a3 2024-11-28T07:42:22,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ed57a6f7561e4bf6b9fb1efc48b2758b is 50, key is test_row_0/C:col10/1732779740579/Put/seqid=0 2024-11-28T07:42:22,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742221_1397 (size=12151) 2024-11-28T07:42:23,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779803130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779803140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,397 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ed57a6f7561e4bf6b9fb1efc48b2758b 2024-11-28T07:42:23,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/d302e66b15e84f239c026cd206030a1f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/d302e66b15e84f239c026cd206030a1f 2024-11-28T07:42:23,406 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/d302e66b15e84f239c026cd206030a1f, entries=150, sequenceid=139, filesize=30.4 K 2024-11-28T07:42:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/b0bd1b88eefa4c9589c02684ab1fa0a3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/b0bd1b88eefa4c9589c02684ab1fa0a3 2024-11-28T07:42:23,410 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/b0bd1b88eefa4c9589c02684ab1fa0a3, entries=150, sequenceid=139, filesize=11.9 K 2024-11-28T07:42:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ed57a6f7561e4bf6b9fb1efc48b2758b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ed57a6f7561e4bf6b9fb1efc48b2758b 2024-11-28T07:42:23,415 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ed57a6f7561e4bf6b9fb1efc48b2758b, entries=150, sequenceid=139, filesize=11.9 K 2024-11-28T07:42:23,415 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 963020bb226d03e8d0fb6ff8c63b3248 in 1293ms, sequenceid=139, compaction requested=false 2024-11-28T07:42:23,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:23,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:23,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-28T07:42:23,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-28T07:42:23,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-28T07:42:23,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7540 sec 2024-11-28T07:42:23,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.7590 sec 2024-11-28T07:42:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:23,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:42:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:23,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:23,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d4ef596fe20c4f83a829770c5e992b0d_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779742797/Put/seqid=0 2024-11-28T07:42:23,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742222_1398 (size=14794) 2024-11-28T07:42:23,456 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:23,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d4ef596fe20c4f83a829770c5e992b0d_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4ef596fe20c4f83a829770c5e992b0d_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:23,460 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/46323de51e5b4fb9a8c360fd6e279588, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:23,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/46323de51e5b4fb9a8c360fd6e279588 is 175, key is test_row_0/A:col10/1732779742797/Put/seqid=0 2024-11-28T07:42:23,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742223_1399 (size=39749) 2024-11-28T07:42:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779803505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779803509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779803620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779803630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-28T07:42:23,810 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:23,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-28T07:42:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T07:42:23,812 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:23,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:23,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:23,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779803830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779803850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:23,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:23,864 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/46323de51e5b4fb9a8c360fd6e279588 2024-11-28T07:42:23,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/5dbc7eddac57436ca1306ca31b573b8e is 50, key is test_row_0/B:col10/1732779742797/Put/seqid=0 2024-11-28T07:42:23,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742224_1400 (size=12151) 2024-11-28T07:42:23,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/5dbc7eddac57436ca1306ca31b573b8e 2024-11-28T07:42:23,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/141bfea25b584246b0f7088f937159a0 is 50, key is test_row_0/C:col10/1732779742797/Put/seqid=0 2024-11-28T07:42:23,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742225_1401 (size=12151) 2024-11-28T07:42:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T07:42:23,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T07:42:23,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:23,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:23,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:23,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:24,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T07:42:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T07:42:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779804150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:24,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779804170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:24,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T07:42:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:24,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:24,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/141bfea25b584246b0f7088f937159a0 2024-11-28T07:42:24,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/46323de51e5b4fb9a8c360fd6e279588 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/46323de51e5b4fb9a8c360fd6e279588 2024-11-28T07:42:24,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/46323de51e5b4fb9a8c360fd6e279588, entries=200, sequenceid=156, filesize=38.8 K 2024-11-28T07:42:24,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/5dbc7eddac57436ca1306ca31b573b8e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/5dbc7eddac57436ca1306ca31b573b8e 2024-11-28T07:42:24,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/5dbc7eddac57436ca1306ca31b573b8e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-28T07:42:24,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/141bfea25b584246b0f7088f937159a0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/141bfea25b584246b0f7088f937159a0 2024-11-28T07:42:24,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/141bfea25b584246b0f7088f937159a0, entries=150, sequenceid=156, filesize=11.9 K 2024-11-28T07:42:24,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 963020bb226d03e8d0fb6ff8c63b3248 in 860ms, sequenceid=156, compaction requested=true 2024-11-28T07:42:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:24,304 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:24,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:24,304 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:24,305 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:24,305 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:24,305 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/A is initiating minor compaction (all files) 2024-11-28T07:42:24,305 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/B is initiating minor compaction (all files) 2024-11-28T07:42:24,305 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/B in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,305 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/A in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,306 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e8bc92cfccf64ab49b2da765ea6b8113, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/b0bd1b88eefa4c9589c02684ab1fa0a3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/5dbc7eddac57436ca1306ca31b573b8e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=35.7 K 2024-11-28T07:42:24,306 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/792996f0098f4d67bab1424880d62628, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/d302e66b15e84f239c026cd206030a1f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/46323de51e5b4fb9a8c360fd6e279588] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=99.7 K 2024-11-28T07:42:24,306 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,306 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/792996f0098f4d67bab1424880d62628, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/d302e66b15e84f239c026cd206030a1f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/46323de51e5b4fb9a8c360fd6e279588] 2024-11-28T07:42:24,306 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e8bc92cfccf64ab49b2da765ea6b8113, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732779739267 2024-11-28T07:42:24,306 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792996f0098f4d67bab1424880d62628, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732779739267 2024-11-28T07:42:24,306 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bd1b88eefa4c9589c02684ab1fa0a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732779740547 2024-11-28T07:42:24,306 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting d302e66b15e84f239c026cd206030a1f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732779740547 2024-11-28T07:42:24,307 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dbc7eddac57436ca1306ca31b573b8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732779742797 2024-11-28T07:42:24,307 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46323de51e5b4fb9a8c360fd6e279588, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732779742797 2024-11-28T07:42:24,312 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:24,314 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:24,314 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128011fe73bd7b64fc1b77dac11f2a78b57_963020bb226d03e8d0fb6ff8c63b3248 store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:24,314 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/3da699783e6e4132a88225bfaa49f7ba is 50, key is test_row_0/B:col10/1732779742797/Put/seqid=0 2024-11-28T07:42:24,316 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128011fe73bd7b64fc1b77dac11f2a78b57_963020bb226d03e8d0fb6ff8c63b3248, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:24,316 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128011fe73bd7b64fc1b77dac11f2a78b57_963020bb226d03e8d0fb6ff8c63b3248 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:24,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742226_1402 (size=12493) 2024-11-28T07:42:24,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742227_1403 (size=4469) 2024-11-28T07:42:24,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-28T07:42:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,426 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:42:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:24,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e1b834b27a6a43bfb2a7fa63f01ede1d_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779743489/Put/seqid=0 2024-11-28T07:42:24,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742228_1404 (size=12304) 2024-11-28T07:42:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T07:42:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:24,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:24,725 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:24,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779804716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:24,726 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/3b5adc750fa54d2bb0a1ae8ff1119f18 is 175, key is test_row_0/A:col10/1732779742797/Put/seqid=0 2024-11-28T07:42:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779804724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:24,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742229_1405 (size=31447) 2024-11-28T07:42:24,733 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/3da699783e6e4132a88225bfaa49f7ba as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/3da699783e6e4132a88225bfaa49f7ba 2024-11-28T07:42:24,741 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/3b5adc750fa54d2bb0a1ae8ff1119f18 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/3b5adc750fa54d2bb0a1ae8ff1119f18 2024-11-28T07:42:24,741 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/B of 963020bb226d03e8d0fb6ff8c63b3248 into 3da699783e6e4132a88225bfaa49f7ba(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:24,741 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:24,742 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/B, priority=13, startTime=1732779744304; duration=0sec 2024-11-28T07:42:24,742 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:24,742 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:B 2024-11-28T07:42:24,742 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:24,743 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:24,743 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/C is initiating minor compaction (all files) 2024-11-28T07:42:24,744 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/C in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:24,744 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/d1e72e5cc9394faa8f72180b3484e589, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ed57a6f7561e4bf6b9fb1efc48b2758b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/141bfea25b584246b0f7088f937159a0] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=35.7 K 2024-11-28T07:42:24,744 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e72e5cc9394faa8f72180b3484e589, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732779739267 2024-11-28T07:42:24,745 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ed57a6f7561e4bf6b9fb1efc48b2758b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732779740547 2024-11-28T07:42:24,745 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 141bfea25b584246b0f7088f937159a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732779742797 2024-11-28T07:42:24,746 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/A of 963020bb226d03e8d0fb6ff8c63b3248 into 3b5adc750fa54d2bb0a1ae8ff1119f18(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:24,746 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:24,746 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/A, priority=13, startTime=1732779744304; duration=0sec 2024-11-28T07:42:24,746 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:24,746 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:A 2024-11-28T07:42:24,753 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:24,754 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ff2584adc27144f993ed53a8093f97f5 is 50, key is test_row_0/C:col10/1732779742797/Put/seqid=0 2024-11-28T07:42:24,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742230_1406 (size=12493) 2024-11-28T07:42:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779804830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779804830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:24,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:24,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:24,843 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128e1b834b27a6a43bfb2a7fa63f01ede1d_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e1b834b27a6a43bfb2a7fa63f01ede1d_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:24,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/49affcca6fe14efeb7b7908671d2ceaf, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:24,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/49affcca6fe14efeb7b7908671d2ceaf is 175, key is test_row_0/A:col10/1732779743489/Put/seqid=0 2024-11-28T07:42:24,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742231_1407 (size=31105) 2024-11-28T07:42:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T07:42:25,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779805040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779805040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,165 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ff2584adc27144f993ed53a8093f97f5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ff2584adc27144f993ed53a8093f97f5 2024-11-28T07:42:25,169 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/C of 963020bb226d03e8d0fb6ff8c63b3248 into ff2584adc27144f993ed53a8093f97f5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:25,169 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:25,169 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/C, priority=13, startTime=1732779744304; duration=0sec 2024-11-28T07:42:25,169 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:25,169 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:C 2024-11-28T07:42:25,249 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/49affcca6fe14efeb7b7908671d2ceaf 2024-11-28T07:42:25,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/fca8f86d0b274ed0aff0642ead168558 is 50, key is test_row_0/B:col10/1732779743489/Put/seqid=0 2024-11-28T07:42:25,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742232_1408 (size=12151) 2024-11-28T07:42:25,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41538 deadline: 1732779805330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,338 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41562 deadline: 1732779805350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779805350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779805350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,357 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,660 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/fca8f86d0b274ed0aff0642ead168558 2024-11-28T07:42:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/73206cfca2e44ddd97c0071023809a16 is 50, key is test_row_0/C:col10/1732779743489/Put/seqid=0 2024-11-28T07:42:25,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742233_1409 (size=12151) 2024-11-28T07:42:25,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779805870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779805870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:25,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T07:42:26,073 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/73206cfca2e44ddd97c0071023809a16 2024-11-28T07:42:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/49affcca6fe14efeb7b7908671d2ceaf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/49affcca6fe14efeb7b7908671d2ceaf 2024-11-28T07:42:26,081 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/49affcca6fe14efeb7b7908671d2ceaf, entries=150, sequenceid=174, filesize=30.4 K 2024-11-28T07:42:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/fca8f86d0b274ed0aff0642ead168558 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/fca8f86d0b274ed0aff0642ead168558 2024-11-28T07:42:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,085 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/fca8f86d0b274ed0aff0642ead168558, entries=150, sequenceid=174, filesize=11.9 K 2024-11-28T07:42:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/73206cfca2e44ddd97c0071023809a16 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/73206cfca2e44ddd97c0071023809a16 2024-11-28T07:42:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,090 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/73206cfca2e44ddd97c0071023809a16, entries=150, sequenceid=174, filesize=11.9 K 2024-11-28T07:42:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,090 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 963020bb226d03e8d0fb6ff8c63b3248 in 1664ms, sequenceid=174, compaction requested=false 2024-11-28T07:42:26,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-28T07:42:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-28T07:42:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-28T07:42:26,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2790 sec 2024-11-28T07:42:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.2830 sec 2024-11-28T07:42:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:26,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:42:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:26,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:26,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112838c9be61874540eb97d9c3c3f1e63c1e_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779744715/Put/seqid=0 2024-11-28T07:42:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742234_1410 (size=12304) 2024-11-28T07:42:26,935 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112838c9be61874540eb97d9c3c3f1e63c1e_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112838c9be61874540eb97d9c3c3f1e63c1e_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:26,949 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c0fdee8df085401eafdbe0546d61cd72, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:26,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c0fdee8df085401eafdbe0546d61cd72 is 175, key is test_row_0/A:col10/1732779744715/Put/seqid=0 2024-11-28T07:42:26,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742235_1411 (size=31105) 2024-11-28T07:42:27,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779807024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779807026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779807140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779807150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c0fdee8df085401eafdbe0546d61cd72 2024-11-28T07:42:27,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/98f42b266ade406aacdc9da0c728c046 is 50, key is test_row_0/B:col10/1732779744715/Put/seqid=0 2024-11-28T07:42:27,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742236_1412 (size=12151) 2024-11-28T07:42:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779807361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779807361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779807680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779807680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:27,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/98f42b266ade406aacdc9da0c728c046 2024-11-28T07:42:27,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ddf038892d5141e1961769bebef16b71 is 50, key is test_row_0/C:col10/1732779744715/Put/seqid=0 2024-11-28T07:42:27,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742237_1413 (size=12151) 2024-11-28T07:42:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-28T07:42:27,981 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:27,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-28T07:42:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T07:42:27,983 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:27,983 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:27,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T07:42:28,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ddf038892d5141e1961769bebef16b71 2024-11-28T07:42:28,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/c0fdee8df085401eafdbe0546d61cd72 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c0fdee8df085401eafdbe0546d61cd72 2024-11-28T07:42:28,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c0fdee8df085401eafdbe0546d61cd72, entries=150, sequenceid=192, filesize=30.4 K 2024-11-28T07:42:28,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/98f42b266ade406aacdc9da0c728c046 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/98f42b266ade406aacdc9da0c728c046 2024-11-28T07:42:28,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/98f42b266ade406aacdc9da0c728c046, entries=150, sequenceid=192, filesize=11.9 K 2024-11-28T07:42:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/ddf038892d5141e1961769bebef16b71 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ddf038892d5141e1961769bebef16b71 2024-11-28T07:42:28,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ddf038892d5141e1961769bebef16b71, entries=150, sequenceid=192, filesize=11.9 K 2024-11-28T07:42:28,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 963020bb226d03e8d0fb6ff8c63b3248 in 1278ms, sequenceid=192, compaction requested=true 2024-11-28T07:42:28,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:28,199 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:28,199 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:28,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:42:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:28,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:28,203 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:28,203 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/A is initiating minor compaction (all files) 2024-11-28T07:42:28,204 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/A in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,204 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/3b5adc750fa54d2bb0a1ae8ff1119f18, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/49affcca6fe14efeb7b7908671d2ceaf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c0fdee8df085401eafdbe0546d61cd72] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=91.5 K 2024-11-28T07:42:28,204 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,204 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/3b5adc750fa54d2bb0a1ae8ff1119f18, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/49affcca6fe14efeb7b7908671d2ceaf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c0fdee8df085401eafdbe0546d61cd72] 2024-11-28T07:42:28,204 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:28,204 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/B is initiating minor compaction (all files) 2024-11-28T07:42:28,204 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/B in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,204 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/3da699783e6e4132a88225bfaa49f7ba, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/fca8f86d0b274ed0aff0642ead168558, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/98f42b266ade406aacdc9da0c728c046] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=35.9 K 2024-11-28T07:42:28,204 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b5adc750fa54d2bb0a1ae8ff1119f18, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732779742797 2024-11-28T07:42:28,205 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da699783e6e4132a88225bfaa49f7ba, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732779742797 2024-11-28T07:42:28,205 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49affcca6fe14efeb7b7908671d2ceaf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732779743489 2024-11-28T07:42:28,205 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting fca8f86d0b274ed0aff0642ead168558, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732779743489 2024-11-28T07:42:28,205 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0fdee8df085401eafdbe0546d61cd72, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732779744715 2024-11-28T07:42:28,206 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f42b266ade406aacdc9da0c728c046, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732779744715 2024-11-28T07:42:28,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128754c006277074dd59a529ada813ed2bd_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779747022/Put/seqid=0 2024-11-28T07:42:28,214 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#B#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:28,215 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/e55864d236bc4d26a1441d92d87b9028 is 50, key is test_row_0/B:col10/1732779744715/Put/seqid=0 2024-11-28T07:42:28,217 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:28,220 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411286b7b43850e9048cda2337d5421d4ee28_963020bb226d03e8d0fb6ff8c63b3248 store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742238_1414 (size=14794) 2024-11-28T07:42:28,225 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411286b7b43850e9048cda2337d5421d4ee28_963020bb226d03e8d0fb6ff8c63b3248, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:28,225 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411286b7b43850e9048cda2337d5421d4ee28_963020bb226d03e8d0fb6ff8c63b3248 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:28,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742240_1416 (size=4469) 2024-11-28T07:42:28,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742239_1415 (size=12595) 2024-11-28T07:42:28,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779808259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:28,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779808261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:28,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:28,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:28,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T07:42:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779808380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779808380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:28,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:28,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:28,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:28,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:28,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T07:42:28,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779808600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779808600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:28,624 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:28,627 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128754c006277074dd59a529ada813ed2bd_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128754c006277074dd59a529ada813ed2bd_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:28,628 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/22267f4d445c434790b2b8a69321a2db, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:28,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/22267f4d445c434790b2b8a69321a2db is 175, key is test_row_0/A:col10/1732779747022/Put/seqid=0 2024-11-28T07:42:28,631 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#A#compaction#353 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:28,632 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/37dcd95d5c694c9584688585a0fecf08 is 175, key is test_row_0/A:col10/1732779744715/Put/seqid=0 2024-11-28T07:42:28,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742241_1417 (size=39749) 2024-11-28T07:42:28,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742242_1418 (size=31549) 2024-11-28T07:42:28,651 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/e55864d236bc4d26a1441d92d87b9028 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e55864d236bc4d26a1441d92d87b9028 2024-11-28T07:42:28,655 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/B of 963020bb226d03e8d0fb6ff8c63b3248 into e55864d236bc4d26a1441d92d87b9028(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:28,655 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:28,655 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/B, priority=13, startTime=1732779748199; duration=0sec 2024-11-28T07:42:28,655 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:28,655 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:B 2024-11-28T07:42:28,655 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:28,656 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:28,656 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/C is initiating minor compaction (all files) 2024-11-28T07:42:28,657 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/C in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,657 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ff2584adc27144f993ed53a8093f97f5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/73206cfca2e44ddd97c0071023809a16, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ddf038892d5141e1961769bebef16b71] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=35.9 K 2024-11-28T07:42:28,657 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ff2584adc27144f993ed53a8093f97f5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732779742797 2024-11-28T07:42:28,657 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 73206cfca2e44ddd97c0071023809a16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732779743489 2024-11-28T07:42:28,658 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf038892d5141e1961769bebef16b71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732779744715 2024-11-28T07:42:28,665 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#C#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:28,666 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/0a7bb7e53c424bcaaaa5da03c8b93c7c is 50, key is test_row_0/C:col10/1732779744715/Put/seqid=0 2024-11-28T07:42:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742243_1419 (size=12595) 2024-11-28T07:42:28,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:28,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:28,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779808920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779808921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:28,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:29,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/22267f4d445c434790b2b8a69321a2db 2024-11-28T07:42:29,042 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/37dcd95d5c694c9584688585a0fecf08 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/37dcd95d5c694c9584688585a0fecf08 2024-11-28T07:42:29,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/85a05a52a1b946ef893d2193e99a6d00 is 50, key is test_row_0/B:col10/1732779747022/Put/seqid=0 2024-11-28T07:42:29,048 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/A of 963020bb226d03e8d0fb6ff8c63b3248 into 37dcd95d5c694c9584688585a0fecf08(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:29,048 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:29,048 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/A, priority=13, startTime=1732779748198; duration=0sec 2024-11-28T07:42:29,048 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:29,048 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:A 2024-11-28T07:42:29,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742244_1420 (size=12151) 2024-11-28T07:42:29,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:29,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,073 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/0a7bb7e53c424bcaaaa5da03c8b93c7c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0a7bb7e53c424bcaaaa5da03c8b93c7c 2024-11-28T07:42:29,077 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/C of 963020bb226d03e8d0fb6ff8c63b3248 into 0a7bb7e53c424bcaaaa5da03c8b93c7c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:29,077 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:29,077 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/C, priority=13, startTime=1732779748199; duration=0sec 2024-11-28T07:42:29,077 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:29,077 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:C 2024-11-28T07:42:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T07:42:29,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:29,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:29,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:29,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:29,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779809440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779809440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:29,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:29,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/85a05a52a1b946ef893d2193e99a6d00 2024-11-28T07:42:29,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/0eaedf22a91041939733543298dfbc70 is 50, key is test_row_0/C:col10/1732779747022/Put/seqid=0 2024-11-28T07:42:29,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742245_1421 (size=12151) 2024-11-28T07:42:29,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/0eaedf22a91041939733543298dfbc70 2024-11-28T07:42:29,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/22267f4d445c434790b2b8a69321a2db as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/22267f4d445c434790b2b8a69321a2db 2024-11-28T07:42:29,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/22267f4d445c434790b2b8a69321a2db, entries=200, sequenceid=210, filesize=38.8 K 2024-11-28T07:42:29,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/85a05a52a1b946ef893d2193e99a6d00 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/85a05a52a1b946ef893d2193e99a6d00 2024-11-28T07:42:29,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/85a05a52a1b946ef893d2193e99a6d00, entries=150, sequenceid=210, filesize=11.9 K 2024-11-28T07:42:29,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/0eaedf22a91041939733543298dfbc70 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0eaedf22a91041939733543298dfbc70 2024-11-28T07:42:29,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0eaedf22a91041939733543298dfbc70, entries=150, sequenceid=210, filesize=11.9 K 2024-11-28T07:42:29,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 963020bb226d03e8d0fb6ff8c63b3248 in 1282ms, sequenceid=210, compaction requested=false 2024-11-28T07:42:29,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:29,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-28T07:42:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:29,517 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:42:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:29,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:29,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280e7e8aa233b14993a1df5f49547fcd51_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779748258/Put/seqid=0 2024-11-28T07:42:29,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742246_1422 (size=12304) 2024-11-28T07:42:29,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:29,933 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280e7e8aa233b14993a1df5f49547fcd51_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280e7e8aa233b14993a1df5f49547fcd51_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:29,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/071294cddaef48b3b3facede7651e9f7, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/071294cddaef48b3b3facede7651e9f7 is 175, key is test_row_0/A:col10/1732779748258/Put/seqid=0 2024-11-28T07:42:29,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742247_1423 (size=31105) 2024-11-28T07:42:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T07:42:30,339 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/071294cddaef48b3b3facede7651e9f7 2024-11-28T07:42:30,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/a9074d2572334e92962e70a8418c10bf is 50, key is test_row_0/B:col10/1732779748258/Put/seqid=0 2024-11-28T07:42:30,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742248_1424 (size=12151) 2024-11-28T07:42:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:30,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:30,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779810513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779810517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:30,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779810631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779810631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:30,749 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/a9074d2572334e92962e70a8418c10bf 2024-11-28T07:42:30,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/7105cba3f485451baacf0f28ba233902 is 50, key is test_row_0/C:col10/1732779748258/Put/seqid=0 2024-11-28T07:42:30,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742249_1425 (size=12151) 2024-11-28T07:42:30,766 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/7105cba3f485451baacf0f28ba233902 2024-11-28T07:42:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/071294cddaef48b3b3facede7651e9f7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/071294cddaef48b3b3facede7651e9f7 2024-11-28T07:42:30,789 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/071294cddaef48b3b3facede7651e9f7, entries=150, sequenceid=227, filesize=30.4 K 2024-11-28T07:42:30,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/a9074d2572334e92962e70a8418c10bf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a9074d2572334e92962e70a8418c10bf 2024-11-28T07:42:30,799 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a9074d2572334e92962e70a8418c10bf, entries=150, sequenceid=227, filesize=11.9 K 2024-11-28T07:42:30,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/7105cba3f485451baacf0f28ba233902 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7105cba3f485451baacf0f28ba233902 2024-11-28T07:42:30,808 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7105cba3f485451baacf0f28ba233902, entries=150, sequenceid=227, filesize=11.9 K 2024-11-28T07:42:30,809 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 963020bb226d03e8d0fb6ff8c63b3248 in 1292ms, sequenceid=227, compaction requested=true 2024-11-28T07:42:30,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:30,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:30,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-28T07:42:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-28T07:42:30,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-28T07:42:30,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8280 sec 2024-11-28T07:42:30,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.8340 sec 2024-11-28T07:42:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:30,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:42:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:30,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:30,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c36355076682409bbe21edfdcead5fa4_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779750515/Put/seqid=0 2024-11-28T07:42:30,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742250_1426 (size=14794) 2024-11-28T07:42:30,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779810905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779810909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:31,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779811020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:31,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779811030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:31,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779811241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:31,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:31,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779811250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:31,270 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:31,273 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c36355076682409bbe21edfdcead5fa4_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c36355076682409bbe21edfdcead5fa4_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:31,274 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/85fb11c827a2429fa1d031d66facfcf5, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:31,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/85fb11c827a2429fa1d031d66facfcf5 is 175, key is test_row_0/A:col10/1732779750515/Put/seqid=0 2024-11-28T07:42:31,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742251_1427 (size=39749) 2024-11-28T07:42:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779811560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:31,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779811570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:31,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/85fb11c827a2429fa1d031d66facfcf5 2024-11-28T07:42:31,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/a2324d4ebd404a0e81f3f31a38eab6ef is 50, key is test_row_0/B:col10/1732779750515/Put/seqid=0 2024-11-28T07:42:31,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742252_1428 (size=12151) 2024-11-28T07:42:31,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41604 deadline: 1732779811780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:31,786 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in 963020bb226d03e8d0fb6ff8c63b3248 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18372 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:31,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779812080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:32,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:32,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/a2324d4ebd404a0e81f3f31a38eab6ef 2024-11-28T07:42:32,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779812090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:32,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:32,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/2a9881482bbb40f48dccfc8487bcf4f1 is 50, key is test_row_0/C:col10/1732779750515/Put/seqid=0 2024-11-28T07:42:32,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742253_1429 (size=12151) 2024-11-28T07:42:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-28T07:42:32,151 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:32,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-28T07:42:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T07:42:32,153 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:32,154 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:32,154 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T07:42:32,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T07:42:32,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:32,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:32,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:32,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T07:42:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:32,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T07:42:32,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/2a9881482bbb40f48dccfc8487bcf4f1 2024-11-28T07:42:32,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/85fb11c827a2429fa1d031d66facfcf5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/85fb11c827a2429fa1d031d66facfcf5 2024-11-28T07:42:32,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/85fb11c827a2429fa1d031d66facfcf5, entries=200, sequenceid=246, filesize=38.8 K 2024-11-28T07:42:32,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/a2324d4ebd404a0e81f3f31a38eab6ef as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a2324d4ebd404a0e81f3f31a38eab6ef 2024-11-28T07:42:32,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a2324d4ebd404a0e81f3f31a38eab6ef, entries=150, sequenceid=246, filesize=11.9 K 2024-11-28T07:42:32,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/2a9881482bbb40f48dccfc8487bcf4f1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2a9881482bbb40f48dccfc8487bcf4f1 2024-11-28T07:42:32,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2a9881482bbb40f48dccfc8487bcf4f1, entries=150, sequenceid=246, filesize=11.9 K 2024-11-28T07:42:32,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 963020bb226d03e8d0fb6ff8c63b3248 in 1666ms, sequenceid=246, compaction requested=true 2024-11-28T07:42:32,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:32,525 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:42:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:32,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:32,525 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:42:32,526 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:42:32,526 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/A is initiating minor compaction (all files) 2024-11-28T07:42:32,526 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:42:32,526 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/B is initiating minor compaction (all files) 2024-11-28T07:42:32,526 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/A in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,526 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/B in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,526 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e55864d236bc4d26a1441d92d87b9028, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/85a05a52a1b946ef893d2193e99a6d00, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a9074d2572334e92962e70a8418c10bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a2324d4ebd404a0e81f3f31a38eab6ef] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=47.9 K 2024-11-28T07:42:32,526 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/37dcd95d5c694c9584688585a0fecf08, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/22267f4d445c434790b2b8a69321a2db, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/071294cddaef48b3b3facede7651e9f7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/85fb11c827a2429fa1d031d66facfcf5] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=138.8 K 2024-11-28T07:42:32,526 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,526 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/37dcd95d5c694c9584688585a0fecf08, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/22267f4d445c434790b2b8a69321a2db, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/071294cddaef48b3b3facede7651e9f7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/85fb11c827a2429fa1d031d66facfcf5] 2024-11-28T07:42:32,527 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e55864d236bc4d26a1441d92d87b9028, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732779744715 2024-11-28T07:42:32,527 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37dcd95d5c694c9584688585a0fecf08, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732779744715 2024-11-28T07:42:32,527 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22267f4d445c434790b2b8a69321a2db, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732779747006 2024-11-28T07:42:32,527 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a05a52a1b946ef893d2193e99a6d00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732779747006 2024-11-28T07:42:32,527 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071294cddaef48b3b3facede7651e9f7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779748231 2024-11-28T07:42:32,527 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a9074d2572334e92962e70a8418c10bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779748231 2024-11-28T07:42:32,528 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85fb11c827a2429fa1d031d66facfcf5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732779750483 2024-11-28T07:42:32,528 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a2324d4ebd404a0e81f3f31a38eab6ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732779750511 2024-11-28T07:42:32,535 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:32,537 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#B#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:32,537 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/cdcddae13960450b94b85ef9689806d4 is 50, key is test_row_0/B:col10/1732779750515/Put/seqid=0 2024-11-28T07:42:32,540 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128a5a93b2ed0714374bbae0ebe62728088_963020bb226d03e8d0fb6ff8c63b3248 store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:32,542 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128a5a93b2ed0714374bbae0ebe62728088_963020bb226d03e8d0fb6ff8c63b3248, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:32,542 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a5a93b2ed0714374bbae0ebe62728088_963020bb226d03e8d0fb6ff8c63b3248 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:32,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742254_1430 (size=12731) 2024-11-28T07:42:32,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742255_1431 (size=4469) 2024-11-28T07:42:32,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-28T07:42:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,613 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:42:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:32,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:32,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128be37944f69ec45779eaa1c212e5bd253_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779750879/Put/seqid=0 2024-11-28T07:42:32,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742256_1432 (size=12404) 2024-11-28T07:42:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T07:42:32,948 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#A#compaction#363 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:32,948 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/297feccba53a4780a23df9d02645842f is 175, key is test_row_0/A:col10/1732779750515/Put/seqid=0 2024-11-28T07:42:32,949 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/cdcddae13960450b94b85ef9689806d4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/cdcddae13960450b94b85ef9689806d4 2024-11-28T07:42:32,954 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/B of 963020bb226d03e8d0fb6ff8c63b3248 into cdcddae13960450b94b85ef9689806d4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:32,954 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:32,954 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/B, priority=12, startTime=1732779752525; duration=0sec 2024-11-28T07:42:32,954 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:32,954 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:B 2024-11-28T07:42:32,954 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:42:32,956 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:42:32,956 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/C is initiating minor compaction (all files) 2024-11-28T07:42:32,956 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/C in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:32,956 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0a7bb7e53c424bcaaaa5da03c8b93c7c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0eaedf22a91041939733543298dfbc70, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7105cba3f485451baacf0f28ba233902, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2a9881482bbb40f48dccfc8487bcf4f1] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=47.9 K 2024-11-28T07:42:32,956 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7bb7e53c424bcaaaa5da03c8b93c7c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732779744715 2024-11-28T07:42:32,957 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eaedf22a91041939733543298dfbc70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732779747006 2024-11-28T07:42:32,957 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 7105cba3f485451baacf0f28ba233902, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779748231 2024-11-28T07:42:32,958 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a9881482bbb40f48dccfc8487bcf4f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732779750511 2024-11-28T07:42:32,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742257_1433 (size=31685) 2024-11-28T07:42:32,969 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#C#compaction#366 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:32,971 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/297feccba53a4780a23df9d02645842f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/297feccba53a4780a23df9d02645842f 2024-11-28T07:42:32,973 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/9eff1a1fc46b428ca33ebb3094f17f4a is 50, key is test_row_0/C:col10/1732779750515/Put/seqid=0 2024-11-28T07:42:32,975 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/A of 963020bb226d03e8d0fb6ff8c63b3248 into 297feccba53a4780a23df9d02645842f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:32,975 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:32,976 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/A, priority=12, startTime=1732779752525; duration=0sec 2024-11-28T07:42:32,976 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:32,976 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:A 2024-11-28T07:42:32,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742258_1434 (size=12731) 2024-11-28T07:42:32,988 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/9eff1a1fc46b428ca33ebb3094f17f4a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/9eff1a1fc46b428ca33ebb3094f17f4a 2024-11-28T07:42:32,996 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/C of 963020bb226d03e8d0fb6ff8c63b3248 into 9eff1a1fc46b428ca33ebb3094f17f4a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:32,996 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:32,996 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/C, priority=12, startTime=1732779752525; duration=0sec 2024-11-28T07:42:32,996 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:32,996 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:C 2024-11-28T07:42:33,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:33,029 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128be37944f69ec45779eaa1c212e5bd253_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128be37944f69ec45779eaa1c212e5bd253_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:33,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e06045b4546d4fa8ada62590a3739501, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:33,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e06045b4546d4fa8ada62590a3739501 is 175, key is test_row_0/A:col10/1732779750879/Put/seqid=0 2024-11-28T07:42:33,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742259_1435 (size=31205) 2024-11-28T07:42:33,037 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e06045b4546d4fa8ada62590a3739501 2024-11-28T07:42:33,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/9f90c27405024f41bfbbd319dbd3c608 is 50, key is test_row_0/B:col10/1732779750879/Put/seqid=0 2024-11-28T07:42:33,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742260_1436 (size=12251) 2024-11-28T07:42:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. as already flushing 2024-11-28T07:42:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779813163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779813174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T07:42:33,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779813280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779813290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,312 INFO [Thread-1646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1646. 2024-11-28T07:42:33,312 DEBUG [Thread-1646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:33,312 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,312 DEBUG [Thread-1646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:33,317 INFO [Thread-1650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1650. 2024-11-28T07:42:33,317 DEBUG [Thread-1650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:33,317 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,317 DEBUG [Thread-1650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,317 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:33,318 INFO [Thread-1652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1652. 2024-11-28T07:42:33,318 DEBUG [Thread-1652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:33,318 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,318 DEBUG [Thread-1652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:33,325 INFO [Thread-1654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1654. 2024-11-28T07:42:33,325 DEBUG [Thread-1654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:33,325 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,325 DEBUG [Thread-1654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:33,326 INFO [Thread-1648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1648. 2024-11-28T07:42:33,327 DEBUG [Thread-1648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:33,327 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,327 DEBUG [Thread-1648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:33,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:33,447 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/9f90c27405024f41bfbbd319dbd3c608 2024-11-28T07:42:33,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/24f5edcd0f004530985bd96916d26917 is 50, key is test_row_0/C:col10/1732779750879/Put/seqid=0 2024-11-28T07:42:33,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742261_1437 (size=12251) 2024-11-28T07:42:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779813500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779813510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,594 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T07:42:33,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41598 deadline: 1732779813811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41532 deadline: 1732779813820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:33,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:42:33,856 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/24f5edcd0f004530985bd96916d26917 2024-11-28T07:42:33,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/e06045b4546d4fa8ada62590a3739501 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e06045b4546d4fa8ada62590a3739501 2024-11-28T07:42:33,863 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e06045b4546d4fa8ada62590a3739501, entries=150, sequenceid=259, filesize=30.5 K 2024-11-28T07:42:33,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/9f90c27405024f41bfbbd319dbd3c608 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/9f90c27405024f41bfbbd319dbd3c608 2024-11-28T07:42:33,866 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/9f90c27405024f41bfbbd319dbd3c608, entries=150, sequenceid=259, filesize=12.0 K 2024-11-28T07:42:33,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/24f5edcd0f004530985bd96916d26917 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/24f5edcd0f004530985bd96916d26917 2024-11-28T07:42:33,870 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/24f5edcd0f004530985bd96916d26917, entries=150, sequenceid=259, filesize=12.0 K 2024-11-28T07:42:33,870 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 963020bb226d03e8d0fb6ff8c63b3248 in 1257ms, sequenceid=259, compaction requested=false 2024-11-28T07:42:33,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:33,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:33,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-28T07:42:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-28T07:42:33,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-28T07:42:33,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-28T07:42:33,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.7220 sec 2024-11-28T07:42:34,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-28T07:42:34,300 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:34,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T07:42:34,322 INFO [Thread-1637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1637. 2024-11-28T07:42:34,322 DEBUG [Thread-1637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:34,322 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:34,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:34,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:34,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:34,322 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:34,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:34,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:34,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:34,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:34,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a24ce64df1364115bd78f6028cf44c10_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_0/A:col10/1732779753162/Put/seqid=0 2024-11-28T07:42:34,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742262_1438 (size=12454) 2024-11-28T07:42:34,331 INFO [Thread-1635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1635. 2024-11-28T07:42:34,332 DEBUG [Thread-1635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:34,332 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:34,332 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:34,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:34,730 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:34,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a24ce64df1364115bd78f6028cf44c10_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a24ce64df1364115bd78f6028cf44c10_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:34,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/7fc878a90fac4db99cdb82aceadfba4e, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:34,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/7fc878a90fac4db99cdb82aceadfba4e is 175, key is test_row_0/A:col10/1732779753162/Put/seqid=0 2024-11-28T07:42:34,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742263_1439 (size=31255) 2024-11-28T07:42:35,138 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/7fc878a90fac4db99cdb82aceadfba4e 2024-11-28T07:42:35,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/1f532d4c20be494484cc5fe5a0f64bbd is 50, key is test_row_0/B:col10/1732779753162/Put/seqid=0 2024-11-28T07:42:35,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742264_1440 (size=12301) 2024-11-28T07:42:35,402 INFO [Thread-1643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1643. 2024-11-28T07:42:35,402 INFO [Thread-1639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1639. 2024-11-28T07:42:35,402 DEBUG [Thread-1643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:35,402 DEBUG [Thread-1639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:35,402 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:35,402 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:35,402 DEBUG [Thread-1639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:35,402 DEBUG [Thread-1643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:35,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:35,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:35,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/1f532d4c20be494484cc5fe5a0f64bbd 2024-11-28T07:42:35,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/f1c59fff22554c52a11e4b8868a2a2d0 is 50, key is test_row_0/C:col10/1732779753162/Put/seqid=0 2024-11-28T07:42:35,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742265_1441 (size=12301) 2024-11-28T07:42:35,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/f1c59fff22554c52a11e4b8868a2a2d0 2024-11-28T07:42:35,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/7fc878a90fac4db99cdb82aceadfba4e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/7fc878a90fac4db99cdb82aceadfba4e 2024-11-28T07:42:35,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/7fc878a90fac4db99cdb82aceadfba4e, entries=150, sequenceid=282, filesize=30.5 K 2024-11-28T07:42:35,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/1f532d4c20be494484cc5fe5a0f64bbd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/1f532d4c20be494484cc5fe5a0f64bbd 2024-11-28T07:42:35,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/1f532d4c20be494484cc5fe5a0f64bbd, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T07:42:35,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/f1c59fff22554c52a11e4b8868a2a2d0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/f1c59fff22554c52a11e4b8868a2a2d0 2024-11-28T07:42:35,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/f1c59fff22554c52a11e4b8868a2a2d0, entries=150, sequenceid=282, filesize=12.0 K 2024-11-28T07:42:35,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for 963020bb226d03e8d0fb6ff8c63b3248 in 1650ms, sequenceid=282, compaction requested=true 2024-11-28T07:42:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:35,971 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 963020bb226d03e8d0fb6ff8c63b3248:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:35,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:35,971 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:35,972 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94145 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:35,972 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:35,972 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/B is initiating minor compaction (all files) 2024-11-28T07:42:35,972 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/A is initiating minor compaction (all files) 2024-11-28T07:42:35,972 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/B in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:35,972 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/A in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:35,972 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/cdcddae13960450b94b85ef9689806d4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/9f90c27405024f41bfbbd319dbd3c608, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/1f532d4c20be494484cc5fe5a0f64bbd] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=36.4 K 2024-11-28T07:42:35,972 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/297feccba53a4780a23df9d02645842f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e06045b4546d4fa8ada62590a3739501, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/7fc878a90fac4db99cdb82aceadfba4e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=91.9 K 2024-11-28T07:42:35,972 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:35,972 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/297feccba53a4780a23df9d02645842f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e06045b4546d4fa8ada62590a3739501, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/7fc878a90fac4db99cdb82aceadfba4e] 2024-11-28T07:42:35,973 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting cdcddae13960450b94b85ef9689806d4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732779750511 2024-11-28T07:42:35,973 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 297feccba53a4780a23df9d02645842f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732779750511 2024-11-28T07:42:35,973 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e06045b4546d4fa8ada62590a3739501, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732779750879 2024-11-28T07:42:35,973 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f90c27405024f41bfbbd319dbd3c608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732779750879 2024-11-28T07:42:35,973 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc878a90fac4db99cdb82aceadfba4e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732779753162 2024-11-28T07:42:35,973 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f532d4c20be494484cc5fe5a0f64bbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732779753162 2024-11-28T07:42:35,979 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:35,980 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112839de9b263ecb47049a2962ce4a3e1e43_963020bb226d03e8d0fb6ff8c63b3248 store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:35,981 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#B#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:35,981 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/19a6b551eaff4f66ae3d057ade682a73 is 50, key is test_row_0/B:col10/1732779753162/Put/seqid=0 2024-11-28T07:42:35,983 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112839de9b263ecb47049a2962ce4a3e1e43_963020bb226d03e8d0fb6ff8c63b3248, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:35,983 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112839de9b263ecb47049a2962ce4a3e1e43_963020bb226d03e8d0fb6ff8c63b3248 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:35,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742266_1442 (size=12983) 2024-11-28T07:42:35,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742267_1443 (size=4469) 2024-11-28T07:42:36,389 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/19a6b551eaff4f66ae3d057ade682a73 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/19a6b551eaff4f66ae3d057ade682a73 2024-11-28T07:42:36,392 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#A#compaction#372 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:36,392 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/9d235a54bda643df9950fe5a3236594c is 175, key is test_row_0/A:col10/1732779753162/Put/seqid=0 2024-11-28T07:42:36,393 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/B of 963020bb226d03e8d0fb6ff8c63b3248 into 19a6b551eaff4f66ae3d057ade682a73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:36,393 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:36,393 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/B, priority=13, startTime=1732779755971; duration=0sec 2024-11-28T07:42:36,393 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:36,393 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:B 2024-11-28T07:42:36,393 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:36,394 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:36,394 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): 963020bb226d03e8d0fb6ff8c63b3248/C is initiating minor compaction (all files) 2024-11-28T07:42:36,394 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 963020bb226d03e8d0fb6ff8c63b3248/C in TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:36,394 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/9eff1a1fc46b428ca33ebb3094f17f4a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/24f5edcd0f004530985bd96916d26917, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/f1c59fff22554c52a11e4b8868a2a2d0] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp, totalSize=36.4 K 2024-11-28T07:42:36,394 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eff1a1fc46b428ca33ebb3094f17f4a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732779750511 2024-11-28T07:42:36,395 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 24f5edcd0f004530985bd96916d26917, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732779750879 2024-11-28T07:42:36,395 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c59fff22554c52a11e4b8868a2a2d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732779753162 2024-11-28T07:42:36,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742268_1444 (size=31937) 2024-11-28T07:42:36,401 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 963020bb226d03e8d0fb6ff8c63b3248#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:36,401 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/7511836a0b8e4afbadabc776e7b0939e is 50, key is test_row_0/C:col10/1732779753162/Put/seqid=0 2024-11-28T07:42:36,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742269_1445 (size=12983) 2024-11-28T07:42:36,799 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/9d235a54bda643df9950fe5a3236594c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9d235a54bda643df9950fe5a3236594c 2024-11-28T07:42:36,803 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/A of 963020bb226d03e8d0fb6ff8c63b3248 into 9d235a54bda643df9950fe5a3236594c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:36,803 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:36,803 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/A, priority=13, startTime=1732779755971; duration=0sec 2024-11-28T07:42:36,803 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:36,803 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:A 2024-11-28T07:42:36,807 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/7511836a0b8e4afbadabc776e7b0939e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7511836a0b8e4afbadabc776e7b0939e 2024-11-28T07:42:36,811 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 963020bb226d03e8d0fb6ff8c63b3248/C of 963020bb226d03e8d0fb6ff8c63b3248 into 7511836a0b8e4afbadabc776e7b0939e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:36,811 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 963020bb226d03e8d0fb6ff8c63b3248: 2024-11-28T07:42:36,811 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., storeName=963020bb226d03e8d0fb6ff8c63b3248/C, priority=13, startTime=1732779755971; duration=0sec 2024-11-28T07:42:36,811 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:36,811 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 963020bb226d03e8d0fb6ff8c63b3248:C 2024-11-28T07:42:37,714 WARN [regionserver/f64cf8bdf19b:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-28T07:42:37,715 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c1297b2de5a14583985b576edf624706, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e82dd7a1706e4de382621a3a0efc55ff, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/746712c24efe43898d9ab276b64a482e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/52e29b97fd92423c812d00be3ad11fd0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/2350fcb3451c482db076a0fda6141a11, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/f64189370df343be938fa52aee4f3a57, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9084f1955f2e4891a3834d46cc6f452d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/792996f0098f4d67bab1424880d62628, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/d302e66b15e84f239c026cd206030a1f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/46323de51e5b4fb9a8c360fd6e279588, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/3b5adc750fa54d2bb0a1ae8ff1119f18, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/49affcca6fe14efeb7b7908671d2ceaf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/37dcd95d5c694c9584688585a0fecf08, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c0fdee8df085401eafdbe0546d61cd72, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/22267f4d445c434790b2b8a69321a2db, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/071294cddaef48b3b3facede7651e9f7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/85fb11c827a2429fa1d031d66facfcf5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/297feccba53a4780a23df9d02645842f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e06045b4546d4fa8ada62590a3739501, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/7fc878a90fac4db99cdb82aceadfba4e] to archive 2024-11-28T07:42:37,716 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:42:37,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c1297b2de5a14583985b576edf624706 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c1297b2de5a14583985b576edf624706 2024-11-28T07:42:37,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e82dd7a1706e4de382621a3a0efc55ff to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e82dd7a1706e4de382621a3a0efc55ff 2024-11-28T07:42:37,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/746712c24efe43898d9ab276b64a482e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/746712c24efe43898d9ab276b64a482e 2024-11-28T07:42:37,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/52e29b97fd92423c812d00be3ad11fd0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/52e29b97fd92423c812d00be3ad11fd0 2024-11-28T07:42:37,721 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/2350fcb3451c482db076a0fda6141a11 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/2350fcb3451c482db076a0fda6141a11 2024-11-28T07:42:37,721 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/f64189370df343be938fa52aee4f3a57 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/f64189370df343be938fa52aee4f3a57 2024-11-28T07:42:37,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9084f1955f2e4891a3834d46cc6f452d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9084f1955f2e4891a3834d46cc6f452d 2024-11-28T07:42:37,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/792996f0098f4d67bab1424880d62628 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/792996f0098f4d67bab1424880d62628 2024-11-28T07:42:37,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/d302e66b15e84f239c026cd206030a1f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/d302e66b15e84f239c026cd206030a1f 2024-11-28T07:42:37,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/46323de51e5b4fb9a8c360fd6e279588 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/46323de51e5b4fb9a8c360fd6e279588 2024-11-28T07:42:37,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/3b5adc750fa54d2bb0a1ae8ff1119f18 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/3b5adc750fa54d2bb0a1ae8ff1119f18 2024-11-28T07:42:37,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/49affcca6fe14efeb7b7908671d2ceaf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/49affcca6fe14efeb7b7908671d2ceaf 2024-11-28T07:42:37,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/37dcd95d5c694c9584688585a0fecf08 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/37dcd95d5c694c9584688585a0fecf08 2024-11-28T07:42:37,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c0fdee8df085401eafdbe0546d61cd72 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/c0fdee8df085401eafdbe0546d61cd72 2024-11-28T07:42:37,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/22267f4d445c434790b2b8a69321a2db to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/22267f4d445c434790b2b8a69321a2db 2024-11-28T07:42:37,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/071294cddaef48b3b3facede7651e9f7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/071294cddaef48b3b3facede7651e9f7 2024-11-28T07:42:37,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/85fb11c827a2429fa1d031d66facfcf5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/85fb11c827a2429fa1d031d66facfcf5 2024-11-28T07:42:37,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/297feccba53a4780a23df9d02645842f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/297feccba53a4780a23df9d02645842f 2024-11-28T07:42:37,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e06045b4546d4fa8ada62590a3739501 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/e06045b4546d4fa8ada62590a3739501 2024-11-28T07:42:37,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/7fc878a90fac4db99cdb82aceadfba4e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/7fc878a90fac4db99cdb82aceadfba4e 2024-11-28T07:42:37,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/104e6cac4bf44a7a9e0e205b40d2c115, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/055995612e86437e81a4ddfd2f5f1654, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/67becaedd9044430871155ad0ef7e4b8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/6dd3944b39ba4d709b653a0473bb52df, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/be3a6af1751049ffa32469560723a299, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/4e3b7f67c49b4f6bbfff917b9ebd0707, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e8bc92cfccf64ab49b2da765ea6b8113, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/bac94b9a8e95452fae7e9a499f7159ba, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/b0bd1b88eefa4c9589c02684ab1fa0a3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/3da699783e6e4132a88225bfaa49f7ba, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/5dbc7eddac57436ca1306ca31b573b8e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/fca8f86d0b274ed0aff0642ead168558, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e55864d236bc4d26a1441d92d87b9028, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/98f42b266ade406aacdc9da0c728c046, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/85a05a52a1b946ef893d2193e99a6d00, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a9074d2572334e92962e70a8418c10bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/cdcddae13960450b94b85ef9689806d4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a2324d4ebd404a0e81f3f31a38eab6ef, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/9f90c27405024f41bfbbd319dbd3c608, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/1f532d4c20be494484cc5fe5a0f64bbd] to archive 2024-11-28T07:42:37,735 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:42:37,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/104e6cac4bf44a7a9e0e205b40d2c115 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/104e6cac4bf44a7a9e0e205b40d2c115 2024-11-28T07:42:37,737 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/055995612e86437e81a4ddfd2f5f1654 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/055995612e86437e81a4ddfd2f5f1654 2024-11-28T07:42:37,738 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/67becaedd9044430871155ad0ef7e4b8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/67becaedd9044430871155ad0ef7e4b8 2024-11-28T07:42:37,739 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/6dd3944b39ba4d709b653a0473bb52df to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/6dd3944b39ba4d709b653a0473bb52df 2024-11-28T07:42:37,740 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/be3a6af1751049ffa32469560723a299 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/be3a6af1751049ffa32469560723a299 2024-11-28T07:42:37,741 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/4e3b7f67c49b4f6bbfff917b9ebd0707 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/4e3b7f67c49b4f6bbfff917b9ebd0707 2024-11-28T07:42:37,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e8bc92cfccf64ab49b2da765ea6b8113 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e8bc92cfccf64ab49b2da765ea6b8113 2024-11-28T07:42:37,742 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/bac94b9a8e95452fae7e9a499f7159ba to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/bac94b9a8e95452fae7e9a499f7159ba 2024-11-28T07:42:37,743 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/b0bd1b88eefa4c9589c02684ab1fa0a3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/b0bd1b88eefa4c9589c02684ab1fa0a3 2024-11-28T07:42:37,744 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/3da699783e6e4132a88225bfaa49f7ba to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/3da699783e6e4132a88225bfaa49f7ba 2024-11-28T07:42:37,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/5dbc7eddac57436ca1306ca31b573b8e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/5dbc7eddac57436ca1306ca31b573b8e 2024-11-28T07:42:37,745 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/fca8f86d0b274ed0aff0642ead168558 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/fca8f86d0b274ed0aff0642ead168558 2024-11-28T07:42:37,746 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e55864d236bc4d26a1441d92d87b9028 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e55864d236bc4d26a1441d92d87b9028 2024-11-28T07:42:37,747 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/98f42b266ade406aacdc9da0c728c046 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/98f42b266ade406aacdc9da0c728c046 2024-11-28T07:42:37,748 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/85a05a52a1b946ef893d2193e99a6d00 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/85a05a52a1b946ef893d2193e99a6d00 2024-11-28T07:42:37,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a9074d2572334e92962e70a8418c10bf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a9074d2572334e92962e70a8418c10bf 2024-11-28T07:42:37,749 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/cdcddae13960450b94b85ef9689806d4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/cdcddae13960450b94b85ef9689806d4 2024-11-28T07:42:37,750 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a2324d4ebd404a0e81f3f31a38eab6ef to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/a2324d4ebd404a0e81f3f31a38eab6ef 2024-11-28T07:42:37,751 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/9f90c27405024f41bfbbd319dbd3c608 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/9f90c27405024f41bfbbd319dbd3c608 2024-11-28T07:42:37,752 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/1f532d4c20be494484cc5fe5a0f64bbd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/1f532d4c20be494484cc5fe5a0f64bbd 2024-11-28T07:42:37,754 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/355067e6072a40c49dcd5f29abea530a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/1116b649683142ceaf6936b58dc8bbd8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/80dbb154abdd4b7e83853d56f042119d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/57bb40dc0bd042d981baaf0c7cfe9d83, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/61be3b2c3a4047418fb2a09fec9941c7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2c91575bffe24e9b9f5df63c1550733d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/d1e72e5cc9394faa8f72180b3484e589, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/db6ea553b0b244cb8a342dce79da6be3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ed57a6f7561e4bf6b9fb1efc48b2758b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ff2584adc27144f993ed53a8093f97f5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/141bfea25b584246b0f7088f937159a0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/73206cfca2e44ddd97c0071023809a16, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0a7bb7e53c424bcaaaa5da03c8b93c7c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ddf038892d5141e1961769bebef16b71, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0eaedf22a91041939733543298dfbc70, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7105cba3f485451baacf0f28ba233902, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/9eff1a1fc46b428ca33ebb3094f17f4a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2a9881482bbb40f48dccfc8487bcf4f1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/24f5edcd0f004530985bd96916d26917, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/f1c59fff22554c52a11e4b8868a2a2d0] to archive 2024-11-28T07:42:37,755 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:42:37,756 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/355067e6072a40c49dcd5f29abea530a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/355067e6072a40c49dcd5f29abea530a 2024-11-28T07:42:37,757 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/1116b649683142ceaf6936b58dc8bbd8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/1116b649683142ceaf6936b58dc8bbd8 2024-11-28T07:42:37,758 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/80dbb154abdd4b7e83853d56f042119d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/80dbb154abdd4b7e83853d56f042119d 2024-11-28T07:42:37,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/57bb40dc0bd042d981baaf0c7cfe9d83 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/57bb40dc0bd042d981baaf0c7cfe9d83 2024-11-28T07:42:37,759 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/61be3b2c3a4047418fb2a09fec9941c7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/61be3b2c3a4047418fb2a09fec9941c7 2024-11-28T07:42:37,760 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2c91575bffe24e9b9f5df63c1550733d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2c91575bffe24e9b9f5df63c1550733d 2024-11-28T07:42:37,761 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/d1e72e5cc9394faa8f72180b3484e589 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/d1e72e5cc9394faa8f72180b3484e589 2024-11-28T07:42:37,762 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/db6ea553b0b244cb8a342dce79da6be3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/db6ea553b0b244cb8a342dce79da6be3 2024-11-28T07:42:37,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ed57a6f7561e4bf6b9fb1efc48b2758b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ed57a6f7561e4bf6b9fb1efc48b2758b 2024-11-28T07:42:37,763 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ff2584adc27144f993ed53a8093f97f5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ff2584adc27144f993ed53a8093f97f5 2024-11-28T07:42:37,764 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/141bfea25b584246b0f7088f937159a0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/141bfea25b584246b0f7088f937159a0 2024-11-28T07:42:37,765 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/73206cfca2e44ddd97c0071023809a16 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/73206cfca2e44ddd97c0071023809a16 2024-11-28T07:42:37,766 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0a7bb7e53c424bcaaaa5da03c8b93c7c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0a7bb7e53c424bcaaaa5da03c8b93c7c 2024-11-28T07:42:37,767 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ddf038892d5141e1961769bebef16b71 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/ddf038892d5141e1961769bebef16b71 2024-11-28T07:42:37,768 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0eaedf22a91041939733543298dfbc70 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/0eaedf22a91041939733543298dfbc70 2024-11-28T07:42:37,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7105cba3f485451baacf0f28ba233902 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7105cba3f485451baacf0f28ba233902 2024-11-28T07:42:37,769 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/9eff1a1fc46b428ca33ebb3094f17f4a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/9eff1a1fc46b428ca33ebb3094f17f4a 2024-11-28T07:42:37,770 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2a9881482bbb40f48dccfc8487bcf4f1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/2a9881482bbb40f48dccfc8487bcf4f1 2024-11-28T07:42:37,771 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/24f5edcd0f004530985bd96916d26917 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/24f5edcd0f004530985bd96916d26917 2024-11-28T07:42:37,772 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f64cf8bdf19b:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/f1c59fff22554c52a11e4b8868a2a2d0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/f1c59fff22554c52a11e4b8868a2a2d0 2024-11-28T07:42:41,861 INFO [Thread-1641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1641. 2024-11-28T07:42:41,861 DEBUG [Thread-1641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:42:41,861 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:41,862 DEBUG [Thread-1641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:41,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1097 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3288 rows 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1099 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3289 rows 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1096 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3285 rows 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1097 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3287 rows 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1098 2024-11-28T07:42:41,862 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3289 rows 2024-11-28T07:42:41,862 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:42:41,862 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:42:41,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:41,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:41,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:42:41,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T07:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T07:42:41,866 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779761866"}]},"ts":"1732779761866"} 2024-11-28T07:42:41,867 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T07:42:41,867 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T07:42:41,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:42:41,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, UNASSIGN}] 2024-11-28T07:42:41,869 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, UNASSIGN 2024-11-28T07:42:41,870 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:41,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, UNASSIGN because future has completed 2024-11-28T07:42:41,871 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:42:41,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T07:42:42,023 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:42,023 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:42:42,023 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 963020bb226d03e8d0fb6ff8c63b3248, disabling compactions & flushes 2024-11-28T07:42:42,023 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:42,023 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:42,023 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. after waiting 0 ms 2024-11-28T07:42:42,023 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:42,023 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 963020bb226d03e8d0fb6ff8c63b3248 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T07:42:42,024 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=A 2024-11-28T07:42:42,024 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:42,024 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=B 2024-11-28T07:42:42,024 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:42,024 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 963020bb226d03e8d0fb6ff8c63b3248, store=C 2024-11-28T07:42:42,024 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:42,029 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bee5298fd1d3438fa71498ae8000dd02_963020bb226d03e8d0fb6ff8c63b3248 is 50, key is test_row_1/A:col10/1732779761860/Put/seqid=0 2024-11-28T07:42:42,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742270_1446 (size=9914) 2024-11-28T07:42:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T07:42:42,432 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:42,435 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bee5298fd1d3438fa71498ae8000dd02_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bee5298fd1d3438fa71498ae8000dd02_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:42,436 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/5daf98f14c3040b18883f4f85a8de84a, store: [table=TestAcidGuarantees family=A region=963020bb226d03e8d0fb6ff8c63b3248] 2024-11-28T07:42:42,437 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/5daf98f14c3040b18883f4f85a8de84a is 175, key is test_row_1/A:col10/1732779761860/Put/seqid=0 2024-11-28T07:42:42,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742271_1447 (size=22561) 2024-11-28T07:42:42,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T07:42:42,840 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/5daf98f14c3040b18883f4f85a8de84a 2024-11-28T07:42:42,846 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/e298b714349449ed87eea66ed2c57ed6 is 50, key is test_row_1/B:col10/1732779761860/Put/seqid=0 2024-11-28T07:42:42,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742272_1448 (size=9857) 2024-11-28T07:42:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T07:42:43,249 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/e298b714349449ed87eea66ed2c57ed6 2024-11-28T07:42:43,256 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/224e8ad4808245a68f19e14701aa7d2f is 50, key is test_row_1/C:col10/1732779761860/Put/seqid=0 2024-11-28T07:42:43,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742273_1449 (size=9857) 2024-11-28T07:42:43,660 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/224e8ad4808245a68f19e14701aa7d2f 2024-11-28T07:42:43,663 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/A/5daf98f14c3040b18883f4f85a8de84a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/5daf98f14c3040b18883f4f85a8de84a 2024-11-28T07:42:43,666 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/5daf98f14c3040b18883f4f85a8de84a, entries=100, sequenceid=292, filesize=22.0 K 2024-11-28T07:42:43,666 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/B/e298b714349449ed87eea66ed2c57ed6 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e298b714349449ed87eea66ed2c57ed6 2024-11-28T07:42:43,669 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e298b714349449ed87eea66ed2c57ed6, entries=100, sequenceid=292, filesize=9.6 K 2024-11-28T07:42:43,669 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/.tmp/C/224e8ad4808245a68f19e14701aa7d2f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/224e8ad4808245a68f19e14701aa7d2f 2024-11-28T07:42:43,673 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/224e8ad4808245a68f19e14701aa7d2f, entries=100, sequenceid=292, filesize=9.6 K 2024-11-28T07:42:43,673 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 963020bb226d03e8d0fb6ff8c63b3248 in 1650ms, sequenceid=292, compaction requested=false 2024-11-28T07:42:43,677 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/recovered.edits/295.seqid, newMaxSeqId=295, maxSeqId=4 2024-11-28T07:42:43,677 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. 2024-11-28T07:42:43,678 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 963020bb226d03e8d0fb6ff8c63b3248: Waiting for close lock at 1732779762023Running coprocessor pre-close hooks at 1732779762023Disabling compacts and flushes for region at 1732779762023Disabling writes for close at 1732779762023Obtaining lock to block concurrent updates at 1732779762023Preparing flush snapshotting stores in 963020bb226d03e8d0fb6ff8c63b3248 at 1732779762023Finished memstore snapshotting TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732779762024 (+1 ms)Flushing stores of TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248. at 1732779762024Flushing 963020bb226d03e8d0fb6ff8c63b3248/A: creating writer at 1732779762024Flushing mob file 963020bb226d03e8d0fb6ff8c63b3248/A: appending metadata at 1732779762028 (+4 ms)Flushing mob file 963020bb226d03e8d0fb6ff8c63b3248/A: closing flushed file at 1732779762028Flushing 963020bb226d03e8d0fb6ff8c63b3248/A: appending metadata at 1732779762436 (+408 ms)Flushing 963020bb226d03e8d0fb6ff8c63b3248/A: closing flushed file at 1732779762436Flushing 963020bb226d03e8d0fb6ff8c63b3248/B: creating writer at 1732779762843 (+407 ms)Flushing 963020bb226d03e8d0fb6ff8c63b3248/B: appending metadata at 1732779762845 (+2 ms)Flushing 963020bb226d03e8d0fb6ff8c63b3248/B: closing flushed file at 1732779762845Flushing 963020bb226d03e8d0fb6ff8c63b3248/C: creating writer at 1732779763253 (+408 ms)Flushing 963020bb226d03e8d0fb6ff8c63b3248/C: appending metadata at 1732779763255 (+2 ms)Flushing 963020bb226d03e8d0fb6ff8c63b3248/C: closing flushed file at 1732779763255Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d2e1d8: reopening flushed file at 1732779763663 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76aa9590: reopening flushed file at 1732779763666 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc875af: reopening flushed file at 1732779763669 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 963020bb226d03e8d0fb6ff8c63b3248 in 1650ms, sequenceid=292, compaction requested=false at 1732779763673 (+4 ms)Writing region close event to WAL at 1732779763675 (+2 ms)Running coprocessor post-close hooks at 1732779763677 (+2 ms)Closed at 1732779763677 2024-11-28T07:42:43,679 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:43,680 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=963020bb226d03e8d0fb6ff8c63b3248, regionState=CLOSED 2024-11-28T07:42:43,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:42:43,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-28T07:42:43,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 963020bb226d03e8d0fb6ff8c63b3248, server=f64cf8bdf19b,34437,1732779635619 in 1.8110 sec 2024-11-28T07:42:43,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-28T07:42:43,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=963020bb226d03e8d0fb6ff8c63b3248, UNASSIGN in 1.8140 sec 2024-11-28T07:42:43,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-28T07:42:43,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8180 sec 2024-11-28T07:42:43,687 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779763687"}]},"ts":"1732779763687"} 2024-11-28T07:42:43,688 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T07:42:43,688 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T07:42:43,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8260 sec 2024-11-28T07:42:44,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-28T07:42:44,021 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:44,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T07:42:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:44,022 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T07:42:44,023 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:44,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,027 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/recovered.edits] 2024-11-28T07:42:44,029 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/5daf98f14c3040b18883f4f85a8de84a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/5daf98f14c3040b18883f4f85a8de84a 2024-11-28T07:42:44,030 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9d235a54bda643df9950fe5a3236594c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/A/9d235a54bda643df9950fe5a3236594c 2024-11-28T07:42:44,032 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/19a6b551eaff4f66ae3d057ade682a73 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/19a6b551eaff4f66ae3d057ade682a73 2024-11-28T07:42:44,033 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e298b714349449ed87eea66ed2c57ed6 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/B/e298b714349449ed87eea66ed2c57ed6 2024-11-28T07:42:44,035 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/224e8ad4808245a68f19e14701aa7d2f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/224e8ad4808245a68f19e14701aa7d2f 2024-11-28T07:42:44,036 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7511836a0b8e4afbadabc776e7b0939e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/C/7511836a0b8e4afbadabc776e7b0939e 2024-11-28T07:42:44,038 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/recovered.edits/295.seqid to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248/recovered.edits/295.seqid 2024-11-28T07:42:44,038 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T07:42:44,039 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T07:42:44,039 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T07:42:44,041 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280e7e8aa233b14993a1df5f49547fcd51_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280e7e8aa233b14993a1df5f49547fcd51_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,042 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281c994461093048be97e07e95199b6f1d_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411281c994461093048be97e07e95199b6f1d_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,043 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f0c21e96e0949bcae5e4518e64f0282_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282f0c21e96e0949bcae5e4518e64f0282_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,043 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283188d1386f074f0f8f1d242a8e246e9a_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411283188d1386f074f0f8f1d242a8e246e9a_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,044 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112838c9be61874540eb97d9c3c3f1e63c1e_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112838c9be61874540eb97d9c3c3f1e63c1e_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,045 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128394efa711cb647edaae908b68e585706_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128394efa711cb647edaae908b68e585706_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,046 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128672d2a89a8054bb3b9257cd5ba5c2655_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128672d2a89a8054bb3b9257cd5ba5c2655_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,046 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128754c006277074dd59a529ada813ed2bd_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128754c006277074dd59a529ada813ed2bd_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,047 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289de4aee35a20472e845cda44f62cbf9a_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289de4aee35a20472e845cda44f62cbf9a_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,048 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a24ce64df1364115bd78f6028cf44c10_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a24ce64df1364115bd78f6028cf44c10_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,049 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128be37944f69ec45779eaa1c212e5bd253_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128be37944f69ec45779eaa1c212e5bd253_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,049 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bee5298fd1d3438fa71498ae8000dd02_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bee5298fd1d3438fa71498ae8000dd02_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,050 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c36355076682409bbe21edfdcead5fa4_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128c36355076682409bbe21edfdcead5fa4_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,051 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4ef596fe20c4f83a829770c5e992b0d_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4ef596fe20c4f83a829770c5e992b0d_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,052 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e1b834b27a6a43bfb2a7fa63f01ede1d_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e1b834b27a6a43bfb2a7fa63f01ede1d_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,053 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e935ff2226614368a2e5559631514c3b_963020bb226d03e8d0fb6ff8c63b3248 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128e935ff2226614368a2e5559631514c3b_963020bb226d03e8d0fb6ff8c63b3248 2024-11-28T07:42:44,053 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T07:42:44,055 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:44,057 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T07:42:44,059 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T07:42:44,060 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:44,060 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T07:42:44,060 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732779764060"}]},"ts":"9223372036854775807"} 2024-11-28T07:42:44,061 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T07:42:44,061 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 963020bb226d03e8d0fb6ff8c63b3248, NAME => 'TestAcidGuarantees,,1732779730195.963020bb226d03e8d0fb6ff8c63b3248.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T07:42:44,061 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T07:42:44,062 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732779764061"}]},"ts":"9223372036854775807"} 2024-11-28T07:42:44,063 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T07:42:44,063 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:44,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-28T07:42:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-28T07:42:44,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T07:42:44,130 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:44,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 421), ProcessCount=11 (was 11), AvailableMemoryMB=6293 (was 6349) 2024-11-28T07:42:44,148 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=6292 2024-11-28T07:42:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:42:44,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:42:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T07:42:44,151 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:42:44,151 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:44,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-28T07:42:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T07:42:44,152 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:42:44,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742274_1450 (size=1061) 2024-11-28T07:42:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T07:42:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T07:42:44,558 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:42:44,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742275_1451 (size=53) 2024-11-28T07:42:44,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T07:42:44,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:42:44,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a71415a0af8a2acbd268f1bf9a3ef418, disabling compactions & flushes 2024-11-28T07:42:44,963 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:44,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:44,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. after waiting 0 ms 2024-11-28T07:42:44,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:44,963 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:44,963 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a71415a0af8a2acbd268f1bf9a3ef418: Waiting for close lock at 1732779764963Disabling compacts and flushes for region at 1732779764963Disabling writes for close at 1732779764963Writing region close event to WAL at 1732779764963Closed at 1732779764963 2024-11-28T07:42:44,964 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:42:44,965 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732779764964"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779764964"}]},"ts":"1732779764964"} 2024-11-28T07:42:44,966 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:42:44,966 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:42:44,967 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779764966"}]},"ts":"1732779764966"} 2024-11-28T07:42:44,968 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T07:42:44,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, ASSIGN}] 2024-11-28T07:42:44,969 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, ASSIGN 2024-11-28T07:42:44,969 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=false 2024-11-28T07:42:45,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T07:42:45,120 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a71415a0af8a2acbd268f1bf9a3ef418, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:45,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, ASSIGN because future has completed 2024-11-28T07:42:45,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:42:45,276 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:45,276 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:42:45,276 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,276 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:42:45,277 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,277 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,278 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,279 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:45,279 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71415a0af8a2acbd268f1bf9a3ef418 columnFamilyName A 2024-11-28T07:42:45,279 DEBUG [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:45,279 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.HStore(327): Store=a71415a0af8a2acbd268f1bf9a3ef418/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:45,279 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,280 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:45,280 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71415a0af8a2acbd268f1bf9a3ef418 columnFamilyName B 2024-11-28T07:42:45,280 DEBUG [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:45,281 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.HStore(327): Store=a71415a0af8a2acbd268f1bf9a3ef418/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:45,281 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,281 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:42:45,282 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71415a0af8a2acbd268f1bf9a3ef418 columnFamilyName C 2024-11-28T07:42:45,282 DEBUG [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:42:45,282 INFO [StoreOpener-a71415a0af8a2acbd268f1bf9a3ef418-1 {}] regionserver.HStore(327): Store=a71415a0af8a2acbd268f1bf9a3ef418/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:42:45,282 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:45,282 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,283 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,283 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,283 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,283 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,284 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:42:45,285 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,286 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:42:45,286 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened a71415a0af8a2acbd268f1bf9a3ef418; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60764343, jitterRate=-0.09454073011875153}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:42:45,286 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:45,287 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for a71415a0af8a2acbd268f1bf9a3ef418: Running coprocessor pre-open hook at 1732779765277Writing region info on filesystem at 1732779765277Initializing all the Stores at 1732779765277Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779765277Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779765277Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779765277Cleaning up temporary data from old regions at 1732779765283 (+6 ms)Running coprocessor post-open hooks at 1732779765286 (+3 ms)Region opened successfully at 1732779765287 (+1 ms) 2024-11-28T07:42:45,287 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., pid=120, masterSystemTime=1732779765273 2024-11-28T07:42:45,289 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:45,289 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:45,289 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a71415a0af8a2acbd268f1bf9a3ef418, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:45,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T07:42:45,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:42:45,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-28T07:42:45,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 in 169 msec 2024-11-28T07:42:45,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-28T07:42:45,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, ASSIGN in 324 msec 2024-11-28T07:42:45,294 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:42:45,295 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779765294"}]},"ts":"1732779765294"} 2024-11-28T07:42:45,295 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T07:42:45,296 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:42:45,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-28T07:42:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-28T07:42:46,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:46,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,303 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4699108c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,304 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,305 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005f4ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,305 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,308 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:42:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,39063,1732779634677 2024-11-28T07:42:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5fbc6871 2024-11-28T07:42:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:42:46,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:42:46,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,310 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31ad88cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa62ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,313 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426a5f00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,314 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44145dc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,316 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4a984, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,317 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d462e3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,318 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67bfe67c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e57e296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667a3b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272f33f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,323 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a8361a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,324 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b72373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f98b82a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,327 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd5e51e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,328 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdabe61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,330 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156eb44a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,331 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b161d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,332 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2463e15b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:42:46,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:42:46,333 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21038c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:42:46,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:42:46,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:42:46,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9072b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:42:46,335 DEBUG [Thread-2037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,335 DEBUG [Thread-2028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,338 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,339 DEBUG [Thread-2026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,341 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:46,341 DEBUG [Thread-2032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,341 DEBUG [Thread-2045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,341 DEBUG [Thread-2034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,342 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-28T07:42:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,344 DEBUG [Thread-2030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T07:42:46,345 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,345 DEBUG [Thread-2039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:42:46,346 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:46,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:46,347 DEBUG [Thread-2039 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,347 DEBUG [Thread-2039 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,348 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:42:46,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:42:46,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,353 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:42:46,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2] 2024-11-28T07:42:46,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:42:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:46,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:46,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/23551abec22840bfb469eca8f4bfea6f is 50, key is test_row_0/A:col10/1732779766357/Put/seqid=0 2024-11-28T07:42:46,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742276_1452 (size=16681) 2024-11-28T07:42:46,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779826396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779826398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779826398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779826399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779826400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T07:42:46,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779826510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779826520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779826521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779826521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779826521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:46,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:46,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T07:42:46,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779826720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779826740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779826740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779826740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779826740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:46,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/23551abec22840bfb469eca8f4bfea6f 2024-11-28T07:42:46,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:46,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:46,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/1392fb20025f48c8924219de586670ba is 50, key is test_row_0/B:col10/1732779766357/Put/seqid=0 2024-11-28T07:42:46,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742277_1453 (size=12001) 2024-11-28T07:42:46,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:46,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T07:42:47,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779827030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779827060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779827060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779827060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779827060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:47,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/1392fb20025f48c8924219de586670ba 2024-11-28T07:42:47,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/5de9eaaa3252442e82cf0493e28e54d8 is 50, key is test_row_0/C:col10/1732779766357/Put/seqid=0 2024-11-28T07:42:47,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742278_1454 (size=12001) 2024-11-28T07:42:47,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:47,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T07:42:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779827550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:47,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779827580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779827580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779827581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779827581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:47,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/5de9eaaa3252442e82cf0493e28e54d8 2024-11-28T07:42:47,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/23551abec22840bfb469eca8f4bfea6f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/23551abec22840bfb469eca8f4bfea6f 2024-11-28T07:42:47,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/23551abec22840bfb469eca8f4bfea6f, entries=250, sequenceid=13, filesize=16.3 K 2024-11-28T07:42:47,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/1392fb20025f48c8924219de586670ba as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/1392fb20025f48c8924219de586670ba 2024-11-28T07:42:47,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/1392fb20025f48c8924219de586670ba, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T07:42:47,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/5de9eaaa3252442e82cf0493e28e54d8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5de9eaaa3252442e82cf0493e28e54d8 2024-11-28T07:42:47,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5de9eaaa3252442e82cf0493e28e54d8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-28T07:42:47,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a71415a0af8a2acbd268f1bf9a3ef418 in 1310ms, sequenceid=13, compaction requested=false 2024-11-28T07:42:47,667 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T07:42:47,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:47,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-28T07:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:47,724 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T07:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:47,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:47,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/2d1ebe989be945cc9e0a7839470c84f0 is 50, key is test_row_0/A:col10/1732779766394/Put/seqid=0 2024-11-28T07:42:47,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742279_1455 (size=12001) 2024-11-28T07:42:48,135 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/2d1ebe989be945cc9e0a7839470c84f0 2024-11-28T07:42:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/ea8fa49dd29e4f1a9f946a93955fcfad is 50, key is test_row_0/B:col10/1732779766394/Put/seqid=0 2024-11-28T07:42:48,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742280_1456 (size=12001) 2024-11-28T07:42:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T07:42:48,547 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/ea8fa49dd29e4f1a9f946a93955fcfad 2024-11-28T07:42:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/81e1d6458ee44092b11f5d4878ac363a is 50, key is test_row_0/C:col10/1732779766394/Put/seqid=0 2024-11-28T07:42:48,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742281_1457 (size=12001) 2024-11-28T07:42:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:48,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779828602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779828605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779828607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779828609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779828608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779828720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779828730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779828730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779828730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779828730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779828940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779828950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779828950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779828951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779828951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:48,972 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/81e1d6458ee44092b11f5d4878ac363a 2024-11-28T07:42:48,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/2d1ebe989be945cc9e0a7839470c84f0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/2d1ebe989be945cc9e0a7839470c84f0 2024-11-28T07:42:48,979 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/2d1ebe989be945cc9e0a7839470c84f0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T07:42:48,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/ea8fa49dd29e4f1a9f946a93955fcfad as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/ea8fa49dd29e4f1a9f946a93955fcfad 2024-11-28T07:42:48,983 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/ea8fa49dd29e4f1a9f946a93955fcfad, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T07:42:48,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/81e1d6458ee44092b11f5d4878ac363a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/81e1d6458ee44092b11f5d4878ac363a 2024-11-28T07:42:48,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/81e1d6458ee44092b11f5d4878ac363a, entries=150, sequenceid=33, filesize=11.7 K 2024-11-28T07:42:48,990 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a71415a0af8a2acbd268f1bf9a3ef418 in 1266ms, sequenceid=33, compaction requested=false 2024-11-28T07:42:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:48,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-28T07:42:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-28T07:42:48,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-28T07:42:48,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6450 sec 2024-11-28T07:42:48,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6520 sec 2024-11-28T07:42:49,192 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:42:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:49,266 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:42:49,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:49,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:49,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:49,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:49,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:49,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:49,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/393912ea1fbc4a5a8512f05a2bcf9622 is 50, key is test_row_0/A:col10/1732779769265/Put/seqid=0 2024-11-28T07:42:49,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742282_1458 (size=14341) 2024-11-28T07:42:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779829300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779829302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779829303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779829303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779829304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779829420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779829420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779829420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779829420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779829430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779829630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779829640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779829640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779829640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779829650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/393912ea1fbc4a5a8512f05a2bcf9622 2024-11-28T07:42:49,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e06f1bb137ee45f5b02f99ab9fa5a202 is 50, key is test_row_0/B:col10/1732779769265/Put/seqid=0 2024-11-28T07:42:49,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742283_1459 (size=12001) 2024-11-28T07:42:49,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779829941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779829950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779829950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779829950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779829970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:49,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:50,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e06f1bb137ee45f5b02f99ab9fa5a202 2024-11-28T07:42:50,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/f2a7a15d51d34ce58559f25b1922d97c is 50, key is test_row_0/C:col10/1732779769265/Put/seqid=0 2024-11-28T07:42:50,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742284_1460 (size=12001) 2024-11-28T07:42:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779830460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:50,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779830470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:50,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779830470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:50,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779830470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:50,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779830490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:50,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/f2a7a15d51d34ce58559f25b1922d97c 2024-11-28T07:42:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-28T07:42:50,510 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:50,510 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:50,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/393912ea1fbc4a5a8512f05a2bcf9622 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/393912ea1fbc4a5a8512f05a2bcf9622 2024-11-28T07:42:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-28T07:42:50,512 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T07:42:50,513 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:50,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:50,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/393912ea1fbc4a5a8512f05a2bcf9622, entries=200, sequenceid=46, filesize=14.0 K 2024-11-28T07:42:50,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e06f1bb137ee45f5b02f99ab9fa5a202 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e06f1bb137ee45f5b02f99ab9fa5a202 2024-11-28T07:42:50,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e06f1bb137ee45f5b02f99ab9fa5a202, entries=150, sequenceid=46, filesize=11.7 K 2024-11-28T07:42:50,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/f2a7a15d51d34ce58559f25b1922d97c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f2a7a15d51d34ce58559f25b1922d97c 2024-11-28T07:42:50,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f2a7a15d51d34ce58559f25b1922d97c, entries=150, sequenceid=46, filesize=11.7 K 2024-11-28T07:42:50,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a71415a0af8a2acbd268f1bf9a3ef418 in 1257ms, sequenceid=46, compaction requested=true 2024-11-28T07:42:50,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:50,524 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:50,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:50,524 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:50,525 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:50,525 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:50,525 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:42:50,525 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:42:50,525 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:50,525 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:50,525 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/1392fb20025f48c8924219de586670ba, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/ea8fa49dd29e4f1a9f946a93955fcfad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e06f1bb137ee45f5b02f99ab9fa5a202] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=35.2 K 2024-11-28T07:42:50,525 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/23551abec22840bfb469eca8f4bfea6f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/2d1ebe989be945cc9e0a7839470c84f0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/393912ea1fbc4a5a8512f05a2bcf9622] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=42.0 K 2024-11-28T07:42:50,525 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1392fb20025f48c8924219de586670ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732779766353 2024-11-28T07:42:50,525 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23551abec22840bfb469eca8f4bfea6f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732779766347 2024-11-28T07:42:50,526 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8fa49dd29e4f1a9f946a93955fcfad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732779766394 2024-11-28T07:42:50,526 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1ebe989be945cc9e0a7839470c84f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732779766394 2024-11-28T07:42:50,526 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 393912ea1fbc4a5a8512f05a2bcf9622, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732779768588 2024-11-28T07:42:50,526 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e06f1bb137ee45f5b02f99ab9fa5a202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732779768603 2024-11-28T07:42:50,534 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:50,535 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/a41d2f7eee0847e88986221d895b363e is 50, key is test_row_0/B:col10/1732779769265/Put/seqid=0 2024-11-28T07:42:50,537 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:50,537 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/db4262a0003b40498f8597b71d04c018 is 50, key is test_row_0/A:col10/1732779769265/Put/seqid=0 2024-11-28T07:42:50,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742286_1462 (size=12104) 2024-11-28T07:42:50,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742285_1461 (size=12104) 2024-11-28T07:42:50,559 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/a41d2f7eee0847e88986221d895b363e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/a41d2f7eee0847e88986221d895b363e 2024-11-28T07:42:50,566 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into a41d2f7eee0847e88986221d895b363e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:50,566 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:50,566 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779770524; duration=0sec 2024-11-28T07:42:50,566 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:50,566 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:42:50,566 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:50,567 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:50,567 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:42:50,567 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:50,567 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5de9eaaa3252442e82cf0493e28e54d8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/81e1d6458ee44092b11f5d4878ac363a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f2a7a15d51d34ce58559f25b1922d97c] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=35.2 K 2024-11-28T07:42:50,567 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5de9eaaa3252442e82cf0493e28e54d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732779766353 2024-11-28T07:42:50,568 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e1d6458ee44092b11f5d4878ac363a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732779766394 2024-11-28T07:42:50,568 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a7a15d51d34ce58559f25b1922d97c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732779768603 2024-11-28T07:42:50,575 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:50,576 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/0f48654adf0b4d22b9f4cc3ead061ef8 is 50, key is test_row_0/C:col10/1732779769265/Put/seqid=0 2024-11-28T07:42:50,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742287_1463 (size=12104) 2024-11-28T07:42:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T07:42:50,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-28T07:42:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:50,666 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:42:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:50,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:50,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/df40ed09c9954a6b9fc68c4e61ee6344 is 50, key is test_row_0/A:col10/1732779769302/Put/seqid=0 2024-11-28T07:42:50,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742288_1464 (size=12001) 2024-11-28T07:42:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T07:42:50,958 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/db4262a0003b40498f8597b71d04c018 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/db4262a0003b40498f8597b71d04c018 2024-11-28T07:42:50,962 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into db4262a0003b40498f8597b71d04c018(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:50,962 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:50,962 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779770523; duration=0sec 2024-11-28T07:42:50,962 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:50,962 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:42:50,983 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/0f48654adf0b4d22b9f4cc3ead061ef8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0f48654adf0b4d22b9f4cc3ead061ef8 2024-11-28T07:42:50,987 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into 0f48654adf0b4d22b9f4cc3ead061ef8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:50,987 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:50,987 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779770524; duration=0sec 2024-11-28T07:42:50,987 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:50,987 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:42:51,078 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/df40ed09c9954a6b9fc68c4e61ee6344 2024-11-28T07:42:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0dca7316985849619b47c9c7681dacf5 is 50, key is test_row_0/B:col10/1732779769302/Put/seqid=0 2024-11-28T07:42:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742289_1465 (size=12001) 2024-11-28T07:42:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T07:42:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:51,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0dca7316985849619b47c9c7681dacf5 2024-11-28T07:42:51,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/1919ba0fd5d24803828ae3a5e41a2b78 is 50, key is test_row_0/C:col10/1732779769302/Put/seqid=0 2024-11-28T07:42:51,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742290_1466 (size=12001) 2024-11-28T07:42:51,499 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/1919ba0fd5d24803828ae3a5e41a2b78 2024-11-28T07:42:51,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/df40ed09c9954a6b9fc68c4e61ee6344 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/df40ed09c9954a6b9fc68c4e61ee6344 2024-11-28T07:42:51,506 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/df40ed09c9954a6b9fc68c4e61ee6344, entries=150, sequenceid=66, filesize=11.7 K 2024-11-28T07:42:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0dca7316985849619b47c9c7681dacf5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0dca7316985849619b47c9c7681dacf5 2024-11-28T07:42:51,509 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0dca7316985849619b47c9c7681dacf5, entries=150, sequenceid=66, filesize=11.7 K 2024-11-28T07:42:51,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/1919ba0fd5d24803828ae3a5e41a2b78 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/1919ba0fd5d24803828ae3a5e41a2b78 2024-11-28T07:42:51,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779831510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779831512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,514 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,516 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/1919ba0fd5d24803828ae3a5e41a2b78, entries=150, sequenceid=66, filesize=11.7 K 2024-11-28T07:42:51,516 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for a71415a0af8a2acbd268f1bf9a3ef418 in 850ms, sequenceid=66, compaction requested=false 2024-11-28T07:42:51,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:51,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:51,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-28T07:42:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-28T07:42:51,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-28T07:42:51,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-28T07:42:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:51,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:42:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:51,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.0120 sec 2024-11-28T07:42:51,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/bf0e1f5a35ec484288d15c1cf8a805d3 is 50, key is test_row_1/A:col10/1732779771511/Put/seqid=0 2024-11-28T07:42:51,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742291_1467 (size=11997) 2024-11-28T07:42:51,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779831555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779831556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779831568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779831620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-28T07:42:51,650 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:51,651 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-28T07:42:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T07:42:51,653 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:51,653 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:51,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:51,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779831670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779831679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779831680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T07:42:51,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:51,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:51,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:51,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779831830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779831880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779831890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779831890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:51,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:51,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/bf0e1f5a35ec484288d15c1cf8a805d3 2024-11-28T07:42:51,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/d65b487c07d3487cb7f0b12fa331a6a1 is 50, key is test_row_1/B:col10/1732779771511/Put/seqid=0 2024-11-28T07:42:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742292_1468 (size=9657) 2024-11-28T07:42:51,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:51,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:51,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T07:42:52,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779832140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779832190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779832210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779832210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:52,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:52,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T07:42:52,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/d65b487c07d3487cb7f0b12fa331a6a1 2024-11-28T07:42:52,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/61f8985fafca4e37a577bfbf0742d58e is 50, key is test_row_1/C:col10/1732779771511/Put/seqid=0 2024-11-28T07:42:52,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742293_1469 (size=9657) 2024-11-28T07:42:52,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:52,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:52,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:52,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779832660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,667 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779832710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:52,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:52,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779832730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779832730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:52,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/61f8985fafca4e37a577bfbf0742d58e 2024-11-28T07:42:52,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/bf0e1f5a35ec484288d15c1cf8a805d3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/bf0e1f5a35ec484288d15c1cf8a805d3 2024-11-28T07:42:52,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/bf0e1f5a35ec484288d15c1cf8a805d3, entries=150, sequenceid=83, filesize=11.7 K 2024-11-28T07:42:52,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/d65b487c07d3487cb7f0b12fa331a6a1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d65b487c07d3487cb7f0b12fa331a6a1 2024-11-28T07:42:52,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d65b487c07d3487cb7f0b12fa331a6a1, entries=100, sequenceid=83, filesize=9.4 K 2024-11-28T07:42:52,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/61f8985fafca4e37a577bfbf0742d58e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/61f8985fafca4e37a577bfbf0742d58e 2024-11-28T07:42:52,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/61f8985fafca4e37a577bfbf0742d58e, entries=100, sequenceid=83, filesize=9.4 K 2024-11-28T07:42:52,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a71415a0af8a2acbd268f1bf9a3ef418 in 1242ms, sequenceid=83, compaction requested=true 2024-11-28T07:42:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:52,764 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:52,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:52,764 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:52,765 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36102 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:52,765 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:52,765 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:42:52,765 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:42:52,765 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,765 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,765 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/db4262a0003b40498f8597b71d04c018, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/df40ed09c9954a6b9fc68c4e61ee6344, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/bf0e1f5a35ec484288d15c1cf8a805d3] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=35.3 K 2024-11-28T07:42:52,765 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/a41d2f7eee0847e88986221d895b363e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0dca7316985849619b47c9c7681dacf5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d65b487c07d3487cb7f0b12fa331a6a1] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=33.0 K 2024-11-28T07:42:52,765 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting db4262a0003b40498f8597b71d04c018, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732779768603 2024-11-28T07:42:52,765 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a41d2f7eee0847e88986221d895b363e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732779768603 2024-11-28T07:42:52,766 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dca7316985849619b47c9c7681dacf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732779769301 2024-11-28T07:42:52,766 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting df40ed09c9954a6b9fc68c4e61ee6344, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732779769301 2024-11-28T07:42:52,766 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d65b487c07d3487cb7f0b12fa331a6a1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779771511 2024-11-28T07:42:52,766 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf0e1f5a35ec484288d15c1cf8a805d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779771510 2024-11-28T07:42:52,773 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:52,773 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/7619c7aecb7f4d049460d779322ffff0 is 50, key is test_row_0/B:col10/1732779769302/Put/seqid=0 2024-11-28T07:42:52,775 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:52,775 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/88dbfbc747f340f0996562502b1f5bb7 is 50, key is test_row_0/A:col10/1732779769302/Put/seqid=0 2024-11-28T07:42:52,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742294_1470 (size=12207) 2024-11-28T07:42:52,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742295_1471 (size=12207) 2024-11-28T07:42:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T07:42:52,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-28T07:42:52,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:52,875 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:42:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:52,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:52,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/72287a34018241d4a8e89e2e76cc0513 is 50, key is test_row_0/A:col10/1732779771555/Put/seqid=0 2024-11-28T07:42:52,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742296_1472 (size=12001) 2024-11-28T07:42:53,182 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/7619c7aecb7f4d049460d779322ffff0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7619c7aecb7f4d049460d779322ffff0 2024-11-28T07:42:53,184 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/88dbfbc747f340f0996562502b1f5bb7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88dbfbc747f340f0996562502b1f5bb7 2024-11-28T07:42:53,195 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into 7619c7aecb7f4d049460d779322ffff0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:53,195 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:53,195 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779772764; duration=0sec 2024-11-28T07:42:53,195 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:53,195 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:42:53,195 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:53,196 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 88dbfbc747f340f0996562502b1f5bb7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:53,196 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:53,196 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779772764; duration=0sec 2024-11-28T07:42:53,196 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:53,196 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:53,196 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:42:53,196 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:42:53,196 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:53,196 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0f48654adf0b4d22b9f4cc3ead061ef8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/1919ba0fd5d24803828ae3a5e41a2b78, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/61f8985fafca4e37a577bfbf0742d58e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=33.0 K 2024-11-28T07:42:53,197 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f48654adf0b4d22b9f4cc3ead061ef8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732779768603 2024-11-28T07:42:53,198 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1919ba0fd5d24803828ae3a5e41a2b78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732779769301 2024-11-28T07:42:53,198 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f8985fafca4e37a577bfbf0742d58e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779771511 2024-11-28T07:42:53,203 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:53,203 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/b59bec35264f4f5ab4ec1b3baba7b62f is 50, key is test_row_0/C:col10/1732779769302/Put/seqid=0 2024-11-28T07:42:53,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742297_1473 (size=12207) 2024-11-28T07:42:53,219 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/b59bec35264f4f5ab4ec1b3baba7b62f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/b59bec35264f4f5ab4ec1b3baba7b62f 2024-11-28T07:42:53,224 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into b59bec35264f4f5ab4ec1b3baba7b62f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:53,224 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:53,224 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779772764; duration=0sec 2024-11-28T07:42:53,224 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:53,225 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:42:53,285 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/72287a34018241d4a8e89e2e76cc0513 2024-11-28T07:42:53,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/761d7af3b94c4dd1b78d416bbfdcd012 is 50, key is test_row_0/B:col10/1732779771555/Put/seqid=0 2024-11-28T07:42:53,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742298_1474 (size=12001) 2024-11-28T07:42:53,298 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/761d7af3b94c4dd1b78d416bbfdcd012 2024-11-28T07:42:53,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/08298e72c5194ce2b7c7cb9add0eb5f5 is 50, key is test_row_0/C:col10/1732779771555/Put/seqid=0 2024-11-28T07:42:53,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742299_1475 (size=12001) 2024-11-28T07:42:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:53,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:53,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779833598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779833670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,676 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,711 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/08298e72c5194ce2b7c7cb9add0eb5f5 2024-11-28T07:42:53,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/72287a34018241d4a8e89e2e76cc0513 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/72287a34018241d4a8e89e2e76cc0513 2024-11-28T07:42:53,718 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/72287a34018241d4a8e89e2e76cc0513, entries=150, sequenceid=100, filesize=11.7 K 2024-11-28T07:42:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779833710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/761d7af3b94c4dd1b78d416bbfdcd012 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/761d7af3b94c4dd1b78d416bbfdcd012 2024-11-28T07:42:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,722 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/761d7af3b94c4dd1b78d416bbfdcd012, entries=150, sequenceid=100, filesize=11.7 K 2024-11-28T07:42:53,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/08298e72c5194ce2b7c7cb9add0eb5f5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/08298e72c5194ce2b7c7cb9add0eb5f5 2024-11-28T07:42:53,726 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/08298e72c5194ce2b7c7cb9add0eb5f5, entries=150, sequenceid=100, filesize=11.7 K 2024-11-28T07:42:53,727 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a71415a0af8a2acbd268f1bf9a3ef418 in 852ms, sequenceid=100, compaction requested=false 2024-11-28T07:42:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-28T07:42:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-28T07:42:53,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-28T07:42:53,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0760 sec 2024-11-28T07:42:53,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.0800 sec 2024-11-28T07:42:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:53,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:42:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:53,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:53,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/fe263e87d6c64908a4fd3ea5647182a7 is 50, key is test_row_0/A:col10/1732779773596/Put/seqid=0 2024-11-28T07:42:53,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742300_1476 (size=14341) 2024-11-28T07:42:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779833774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779833780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779833787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-28T07:42:53,800 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:53,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-28T07:42:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T07:42:53,802 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:53,803 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:53,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779833890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779833900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779833900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T07:42:53,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779833930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:53,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:53,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:53,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:53,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779834110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779834110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779834110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T07:42:54,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/fe263e87d6c64908a4fd3ea5647182a7 2024-11-28T07:42:54,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e1df8bc6a3f04930b2880acdd342bb1f is 50, key is test_row_0/B:col10/1732779773596/Put/seqid=0 2024-11-28T07:42:54,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742301_1477 (size=12001) 2024-11-28T07:42:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779834250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T07:42:54,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779834430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779834430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779834430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e1df8bc6a3f04930b2880acdd342bb1f 2024-11-28T07:42:54,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/a908083c0405487d8da34a4e507f6ecd is 50, key is test_row_0/C:col10/1732779773596/Put/seqid=0 2024-11-28T07:42:54,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742302_1478 (size=12001) 2024-11-28T07:42:54,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779834760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T07:42:54,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779834940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779834940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779834950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:54,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:54,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/a908083c0405487d8da34a4e507f6ecd 2024-11-28T07:42:54,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/fe263e87d6c64908a4fd3ea5647182a7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/fe263e87d6c64908a4fd3ea5647182a7 2024-11-28T07:42:54,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/fe263e87d6c64908a4fd3ea5647182a7, entries=200, sequenceid=119, filesize=14.0 K 2024-11-28T07:42:54,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e1df8bc6a3f04930b2880acdd342bb1f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e1df8bc6a3f04930b2880acdd342bb1f 2024-11-28T07:42:54,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e1df8bc6a3f04930b2880acdd342bb1f, entries=150, sequenceid=119, filesize=11.7 K 2024-11-28T07:42:54,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/a908083c0405487d8da34a4e507f6ecd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/a908083c0405487d8da34a4e507f6ecd 2024-11-28T07:42:54,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/a908083c0405487d8da34a4e507f6ecd, entries=150, sequenceid=119, filesize=11.7 K 2024-11-28T07:42:54,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a71415a0af8a2acbd268f1bf9a3ef418 in 1245ms, sequenceid=119, compaction requested=true 2024-11-28T07:42:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:54,978 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:54,978 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:54,979 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:54,979 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:54,979 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:42:54,979 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:42:54,979 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,979 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:54,979 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7619c7aecb7f4d049460d779322ffff0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/761d7af3b94c4dd1b78d416bbfdcd012, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e1df8bc6a3f04930b2880acdd342bb1f] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=35.4 K 2024-11-28T07:42:54,979 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88dbfbc747f340f0996562502b1f5bb7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/72287a34018241d4a8e89e2e76cc0513, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/fe263e87d6c64908a4fd3ea5647182a7] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=37.6 K 2024-11-28T07:42:54,980 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 7619c7aecb7f4d049460d779322ffff0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779769302 2024-11-28T07:42:54,980 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88dbfbc747f340f0996562502b1f5bb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779769302 2024-11-28T07:42:54,980 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72287a34018241d4a8e89e2e76cc0513, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732779771553 2024-11-28T07:42:54,980 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 761d7af3b94c4dd1b78d416bbfdcd012, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732779771553 2024-11-28T07:42:54,980 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe263e87d6c64908a4fd3ea5647182a7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732779773572 2024-11-28T07:42:54,980 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e1df8bc6a3f04930b2880acdd342bb1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732779773584 2024-11-28T07:42:54,986 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:54,987 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/7bf5447576b346b0b7face8960d40fad is 50, key is test_row_0/B:col10/1732779773596/Put/seqid=0 2024-11-28T07:42:54,988 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:54,988 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/88f1bb86975849bfb13637148ae8a46b is 50, key is test_row_0/A:col10/1732779773596/Put/seqid=0 2024-11-28T07:42:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742303_1479 (size=12309) 2024-11-28T07:42:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742304_1480 (size=12309) 2024-11-28T07:42:55,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-28T07:42:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:55,026 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:42:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:55,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e2494938155e4fe7a2be0f3df3b25622 is 50, key is test_row_0/A:col10/1732779773772/Put/seqid=0 2024-11-28T07:42:55,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742305_1481 (size=12151) 2024-11-28T07:42:55,036 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e2494938155e4fe7a2be0f3df3b25622 2024-11-28T07:42:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0002089b1c2a462bb81a0dbd9992b0ce is 50, key is test_row_0/B:col10/1732779773772/Put/seqid=0 2024-11-28T07:42:55,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742306_1482 (size=12151) 2024-11-28T07:42:55,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T07:42:55,113 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T07:42:55,398 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/7bf5447576b346b0b7face8960d40fad as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7bf5447576b346b0b7face8960d40fad 2024-11-28T07:42:55,403 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into 7bf5447576b346b0b7face8960d40fad(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:55,403 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:55,403 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779774978; duration=0sec 2024-11-28T07:42:55,403 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:55,403 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:42:55,403 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:55,404 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:55,404 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:42:55,404 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:55,404 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/b59bec35264f4f5ab4ec1b3baba7b62f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/08298e72c5194ce2b7c7cb9add0eb5f5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/a908083c0405487d8da34a4e507f6ecd] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=35.4 K 2024-11-28T07:42:55,405 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting b59bec35264f4f5ab4ec1b3baba7b62f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732779769302 2024-11-28T07:42:55,405 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/88f1bb86975849bfb13637148ae8a46b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88f1bb86975849bfb13637148ae8a46b 2024-11-28T07:42:55,405 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 08298e72c5194ce2b7c7cb9add0eb5f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732779771553 2024-11-28T07:42:55,405 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a908083c0405487d8da34a4e507f6ecd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732779773584 2024-11-28T07:42:55,413 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 88f1bb86975849bfb13637148ae8a46b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:55,413 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:55,413 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779774978; duration=0sec 2024-11-28T07:42:55,413 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:55,413 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:42:55,421 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:55,422 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/c4346dfe09644b74aa9ef03086b8e75f is 50, key is test_row_0/C:col10/1732779773596/Put/seqid=0 2024-11-28T07:42:55,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742307_1483 (size=12309) 2024-11-28T07:42:55,450 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0002089b1c2a462bb81a0dbd9992b0ce 2024-11-28T07:42:55,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/c049e0a97a3549c49b76521d87fcd829 is 50, key is test_row_0/C:col10/1732779773772/Put/seqid=0 2024-11-28T07:42:55,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742308_1484 (size=12151) 2024-11-28T07:42:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:55,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:55,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779835779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779835781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:55,834 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/c4346dfe09644b74aa9ef03086b8e75f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c4346dfe09644b74aa9ef03086b8e75f 2024-11-28T07:42:55,838 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into c4346dfe09644b74aa9ef03086b8e75f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:55,838 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:55,838 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779774978; duration=0sec 2024-11-28T07:42:55,838 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:55,838 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:42:55,862 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/c049e0a97a3549c49b76521d87fcd829 2024-11-28T07:42:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e2494938155e4fe7a2be0f3df3b25622 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2494938155e4fe7a2be0f3df3b25622 2024-11-28T07:42:55,869 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2494938155e4fe7a2be0f3df3b25622, entries=150, sequenceid=135, filesize=11.9 K 2024-11-28T07:42:55,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0002089b1c2a462bb81a0dbd9992b0ce as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0002089b1c2a462bb81a0dbd9992b0ce 2024-11-28T07:42:55,873 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0002089b1c2a462bb81a0dbd9992b0ce, entries=150, sequenceid=135, filesize=11.9 K 2024-11-28T07:42:55,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/c049e0a97a3549c49b76521d87fcd829 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c049e0a97a3549c49b76521d87fcd829 2024-11-28T07:42:55,877 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c049e0a97a3549c49b76521d87fcd829, entries=150, sequenceid=135, filesize=11.9 K 2024-11-28T07:42:55,878 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a71415a0af8a2acbd268f1bf9a3ef418 in 852ms, sequenceid=135, compaction requested=false 2024-11-28T07:42:55,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:55,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:55,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-28T07:42:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-28T07:42:55,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-28T07:42:55,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0770 sec 2024-11-28T07:42:55,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0820 sec 2024-11-28T07:42:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:55,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:42:55,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:55,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:55,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:55,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:55,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:55,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:55,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/7633bf6552ba48fb98d1ded91ec748fc is 50, key is test_row_0/A:col10/1732779775779/Put/seqid=0 2024-11-28T07:42:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742309_1485 (size=14541) 2024-11-28T07:42:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-28T07:42:55,950 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:42:55,950 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:42:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-28T07:42:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T07:42:55,952 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:42:55,953 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:42:55,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:42:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779835960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:55,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779835960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:55,967 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:55,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779835966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:55,967 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779835967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:55,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779835970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:55,979 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:55,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T07:42:56,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779836070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779836079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:56,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T07:42:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T07:42:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T07:42:56,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779836290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:56,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/7633bf6552ba48fb98d1ded91ec748fc 2024-11-28T07:42:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779836300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:56,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/24ad87e141c84f32b150c8736d5dcd3a is 50, key is test_row_0/B:col10/1732779775779/Put/seqid=0 2024-11-28T07:42:56,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742310_1486 (size=12151) 2024-11-28T07:42:56,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T07:42:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T07:42:56,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:56,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T07:42:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:56,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779836610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:56,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779836610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:56,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T07:42:56,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:56,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/24ad87e141c84f32b150c8736d5dcd3a 2024-11-28T07:42:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:42:56,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/383e6c4e1ae645b2a6a951080203bd01 is 50, key is test_row_0/C:col10/1732779775779/Put/seqid=0 2024-11-28T07:42:56,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742311_1487 (size=12151) 2024-11-28T07:42:56,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/383e6c4e1ae645b2a6a951080203bd01 2024-11-28T07:42:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/7633bf6552ba48fb98d1ded91ec748fc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/7633bf6552ba48fb98d1ded91ec748fc 2024-11-28T07:42:56,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/7633bf6552ba48fb98d1ded91ec748fc, entries=200, sequenceid=155, filesize=14.2 K 2024-11-28T07:42:56,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/24ad87e141c84f32b150c8736d5dcd3a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/24ad87e141c84f32b150c8736d5dcd3a 2024-11-28T07:42:56,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/24ad87e141c84f32b150c8736d5dcd3a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T07:42:56,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/383e6c4e1ae645b2a6a951080203bd01 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/383e6c4e1ae645b2a6a951080203bd01 2024-11-28T07:42:56,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/383e6c4e1ae645b2a6a951080203bd01, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T07:42:56,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for a71415a0af8a2acbd268f1bf9a3ef418 in 851ms, sequenceid=155, compaction requested=true 2024-11-28T07:42:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:56,747 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:56,747 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:56,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:42:56,748 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,748 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,748 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7bf5447576b346b0b7face8960d40fad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0002089b1c2a462bb81a0dbd9992b0ce, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/24ad87e141c84f32b150c8736d5dcd3a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=35.8 K 2024-11-28T07:42:56,748 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88f1bb86975849bfb13637148ae8a46b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2494938155e4fe7a2be0f3df3b25622, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/7633bf6552ba48fb98d1ded91ec748fc] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=38.1 K 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf5447576b346b0b7face8960d40fad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732779773584 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f1bb86975849bfb13637148ae8a46b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732779773584 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0002089b1c2a462bb81a0dbd9992b0ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732779773772 2024-11-28T07:42:56,748 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2494938155e4fe7a2be0f3df3b25622, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732779773772 2024-11-28T07:42:56,749 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ad87e141c84f32b150c8736d5dcd3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779775768 2024-11-28T07:42:56,749 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7633bf6552ba48fb98d1ded91ec748fc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779775768 2024-11-28T07:42:56,754 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:56,755 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/15d301aa6d9e4ad4a744ab624b87c75f is 50, key is test_row_0/A:col10/1732779775779/Put/seqid=0 2024-11-28T07:42:56,757 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:56,757 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/9a6e529c7a72440f8268635675ee74ed is 50, key is test_row_0/B:col10/1732779775779/Put/seqid=0 2024-11-28T07:42:56,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742312_1488 (size=12561) 2024-11-28T07:42:56,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742313_1489 (size=12561) 2024-11-28T07:42:56,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-28T07:42:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:56,869 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:42:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:56,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:56,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e9674d59b9784c48bc85aaafc04982ea is 50, key is test_row_0/A:col10/1732779775964/Put/seqid=0 2024-11-28T07:42:56,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742314_1490 (size=12151) 2024-11-28T07:42:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T07:42:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:57,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:42:57,170 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/15d301aa6d9e4ad4a744ab624b87c75f as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/15d301aa6d9e4ad4a744ab624b87c75f 2024-11-28T07:42:57,170 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/9a6e529c7a72440f8268635675ee74ed as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9a6e529c7a72440f8268635675ee74ed 2024-11-28T07:42:57,173 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 15d301aa6d9e4ad4a744ab624b87c75f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:57,173 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:57,173 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into 9a6e529c7a72440f8268635675ee74ed(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:57,173 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779776747; duration=0sec 2024-11-28T07:42:57,173 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:57,173 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779776747; duration=0sec 2024-11-28T07:42:57,173 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:57,174 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:42:57,174 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:57,174 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:42:57,174 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:57,174 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:57,174 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:42:57,174 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:57,174 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c4346dfe09644b74aa9ef03086b8e75f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c049e0a97a3549c49b76521d87fcd829, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/383e6c4e1ae645b2a6a951080203bd01] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=35.8 K 2024-11-28T07:42:57,175 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4346dfe09644b74aa9ef03086b8e75f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732779773584 2024-11-28T07:42:57,175 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c049e0a97a3549c49b76521d87fcd829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732779773772 2024-11-28T07:42:57,175 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 383e6c4e1ae645b2a6a951080203bd01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779775768 2024-11-28T07:42:57,182 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:57,182 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/54c0ea4776324d4f9d8ccd801730d13c is 50, key is test_row_0/C:col10/1732779775779/Put/seqid=0 2024-11-28T07:42:57,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742315_1491 (size=12561) 2024-11-28T07:42:57,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779837180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779837181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,277 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e9674d59b9784c48bc85aaafc04982ea 2024-11-28T07:42:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/2aee833bae1f4be5b7974a71eb833868 is 50, key is test_row_0/B:col10/1732779775964/Put/seqid=0 2024-11-28T07:42:57,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742316_1492 (size=12151) 2024-11-28T07:42:57,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779837291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779837300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779837500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779837510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,589 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/54c0ea4776324d4f9d8ccd801730d13c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54c0ea4776324d4f9d8ccd801730d13c 2024-11-28T07:42:57,593 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into 54c0ea4776324d4f9d8ccd801730d13c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:57,593 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:57,593 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779776747; duration=0sec 2024-11-28T07:42:57,594 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:57,594 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:42:57,687 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/2aee833bae1f4be5b7974a71eb833868 2024-11-28T07:42:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/54e9a03386c34b93b12e01055b957eaa is 50, key is test_row_0/C:col10/1732779775964/Put/seqid=0 2024-11-28T07:42:57,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742317_1493 (size=12151) 2024-11-28T07:42:57,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779837810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779837820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779837980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,983 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779837990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:57,996 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779838000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:58,004 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:42:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,097 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/54e9a03386c34b93b12e01055b957eaa 2024-11-28T07:42:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T07:42:58,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e9674d59b9784c48bc85aaafc04982ea as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e9674d59b9784c48bc85aaafc04982ea 2024-11-28T07:42:58,105 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e9674d59b9784c48bc85aaafc04982ea, entries=150, sequenceid=171, filesize=11.9 K 2024-11-28T07:42:58,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/2aee833bae1f4be5b7974a71eb833868 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2aee833bae1f4be5b7974a71eb833868 2024-11-28T07:42:58,108 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2aee833bae1f4be5b7974a71eb833868, entries=150, sequenceid=171, filesize=11.9 K 2024-11-28T07:42:58,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/54e9a03386c34b93b12e01055b957eaa as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54e9a03386c34b93b12e01055b957eaa 2024-11-28T07:42:58,112 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54e9a03386c34b93b12e01055b957eaa, entries=150, sequenceid=171, filesize=11.9 K 2024-11-28T07:42:58,113 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a71415a0af8a2acbd268f1bf9a3ef418 in 1244ms, sequenceid=171, compaction requested=false 2024-11-28T07:42:58,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:58,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:58,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-28T07:42:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-28T07:42:58,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-28T07:42:58,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-28T07:42:58,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.1660 sec 2024-11-28T07:42:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:42:58,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:42:58,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:42:58,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:58,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:42:58,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:42:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:42:58,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/78f93030a7fe479baa35ed304b82f25d is 50, key is test_row_0/A:col10/1732779777179/Put/seqid=0 2024-11-28T07:42:58,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742318_1494 (size=12151) 2024-11-28T07:42:58,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779838381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779838381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779838490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779838500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779838700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779838720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:58,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/78f93030a7fe479baa35ed304b82f25d 2024-11-28T07:42:58,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/b149b5e17ae84a1d8a8a09645a0ae626 is 50, key is test_row_0/B:col10/1732779777179/Put/seqid=0 2024-11-28T07:42:58,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742319_1495 (size=12151) 2024-11-28T07:42:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:59,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779839021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:59,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779839040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:59,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/b149b5e17ae84a1d8a8a09645a0ae626 2024-11-28T07:42:59,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/e08f95a0953b4a23b241aff64d4db9ff is 50, key is test_row_0/C:col10/1732779777179/Put/seqid=0 2024-11-28T07:42:59,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742320_1496 (size=12151) 2024-11-28T07:42:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779839530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:59,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/e08f95a0953b4a23b241aff64d4db9ff 2024-11-28T07:42:59,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/78f93030a7fe479baa35ed304b82f25d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/78f93030a7fe479baa35ed304b82f25d 2024-11-28T07:42:59,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:42:59,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/78f93030a7fe479baa35ed304b82f25d, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T07:42:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779839560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:42:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:42:59,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:42:59,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/b149b5e17ae84a1d8a8a09645a0ae626 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b149b5e17ae84a1d8a8a09645a0ae626 2024-11-28T07:42:59,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b149b5e17ae84a1d8a8a09645a0ae626, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T07:42:59,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/e08f95a0953b4a23b241aff64d4db9ff as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e08f95a0953b4a23b241aff64d4db9ff 2024-11-28T07:42:59,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e08f95a0953b4a23b241aff64d4db9ff, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T07:42:59,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a71415a0af8a2acbd268f1bf9a3ef418 in 1244ms, sequenceid=191, compaction requested=true 2024-11-28T07:42:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:42:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:42:59,576 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:42:59,576 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:42:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:59,577 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:59,577 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:42:59,577 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:42:59,577 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:42:59,577 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:59,577 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:42:59,577 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9a6e529c7a72440f8268635675ee74ed, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2aee833bae1f4be5b7974a71eb833868, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b149b5e17ae84a1d8a8a09645a0ae626] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.0 K 2024-11-28T07:42:59,577 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/15d301aa6d9e4ad4a744ab624b87c75f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e9674d59b9784c48bc85aaafc04982ea, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/78f93030a7fe479baa35ed304b82f25d] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.0 K 2024-11-28T07:42:59,577 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6e529c7a72440f8268635675ee74ed, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779775768 2024-11-28T07:42:59,578 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d301aa6d9e4ad4a744ab624b87c75f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779775768 2024-11-28T07:42:59,578 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aee833bae1f4be5b7974a71eb833868, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732779775914 2024-11-28T07:42:59,578 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9674d59b9784c48bc85aaafc04982ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732779775914 2024-11-28T07:42:59,578 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78f93030a7fe479baa35ed304b82f25d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779777177 2024-11-28T07:42:59,578 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting b149b5e17ae84a1d8a8a09645a0ae626, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779777177 2024-11-28T07:42:59,584 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:59,584 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/5c846b788de144e1a3bad1e726d1dea8 is 50, key is test_row_0/A:col10/1732779777179/Put/seqid=0 2024-11-28T07:42:59,586 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:42:59,587 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0bbc41034f274567a638912473ff0ce9 is 50, key is test_row_0/B:col10/1732779777179/Put/seqid=0 2024-11-28T07:42:59,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742321_1497 (size=12663) 2024-11-28T07:42:59,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742322_1498 (size=12663) 2024-11-28T07:42:59,994 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/5c846b788de144e1a3bad1e726d1dea8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/5c846b788de144e1a3bad1e726d1dea8 2024-11-28T07:42:59,995 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0bbc41034f274567a638912473ff0ce9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0bbc41034f274567a638912473ff0ce9 2024-11-28T07:42:59,999 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 5c846b788de144e1a3bad1e726d1dea8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:59,999 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:59,999 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779779576; duration=0sec 2024-11-28T07:42:59,999 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into 0bbc41034f274567a638912473ff0ce9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:42:59,999 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:42:59,999 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779779576; duration=0sec 2024-11-28T07:42:59,999 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:42:59,999 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:42:59,999 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:42:59,999 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:42:59,999 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:00,000 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:00,000 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:43:00,000 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:00,000 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54c0ea4776324d4f9d8ccd801730d13c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54e9a03386c34b93b12e01055b957eaa, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e08f95a0953b4a23b241aff64d4db9ff] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.0 K 2024-11-28T07:43:00,000 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c0ea4776324d4f9d8ccd801730d13c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779775768 2024-11-28T07:43:00,001 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54e9a03386c34b93b12e01055b957eaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732779775914 2024-11-28T07:43:00,001 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08f95a0953b4a23b241aff64d4db9ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779777177 2024-11-28T07:43:00,010 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:00,011 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/743f594ad0ab425db590a1bcd36ec5e4 is 50, key is test_row_0/C:col10/1732779777179/Put/seqid=0 2024-11-28T07:43:00,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742323_1499 (size=12663) 2024-11-28T07:43:00,023 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/743f594ad0ab425db590a1bcd36ec5e4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/743f594ad0ab425db590a1bcd36ec5e4 2024-11-28T07:43:00,027 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into 743f594ad0ab425db590a1bcd36ec5e4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:00,027 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:00,027 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779779576; duration=0sec 2024-11-28T07:43:00,027 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:00,027 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:43:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-28T07:43:00,120 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:00,121 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-28T07:43:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T07:43:00,123 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:00,123 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:00,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T07:43:00,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-28T07:43:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:00,276 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:43:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:00,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:00,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e2b73ae9b4b6461591c3b8d47e0cd8f8 is 50, key is test_row_0/A:col10/1732779778380/Put/seqid=0 2024-11-28T07:43:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742324_1500 (size=12151) 2024-11-28T07:43:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T07:43:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:00,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:00,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779840615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:00,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779840619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:00,687 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e2b73ae9b4b6461591c3b8d47e0cd8f8 2024-11-28T07:43:00,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/47d852db43c549afa13e26594a393b9d is 50, key is test_row_0/B:col10/1732779778380/Put/seqid=0 2024-11-28T07:43:00,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742325_1501 (size=12151) 2024-11-28T07:43:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779840730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779840730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T07:43:00,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:00,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779840950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:00,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779840950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:01,099 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/47d852db43c549afa13e26594a393b9d 2024-11-28T07:43:01,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/6bfd13e9adee4b53a786c13c2f8cc21c is 50, key is test_row_0/C:col10/1732779778380/Put/seqid=0 2024-11-28T07:43:01,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742326_1502 (size=12151) 2024-11-28T07:43:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T07:43:01,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779841261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:01,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:01,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:01,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779841261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:01,548 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/6bfd13e9adee4b53a786c13c2f8cc21c 2024-11-28T07:43:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/e2b73ae9b4b6461591c3b8d47e0cd8f8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2b73ae9b4b6461591c3b8d47e0cd8f8 2024-11-28T07:43:01,561 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2b73ae9b4b6461591c3b8d47e0cd8f8, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T07:43:01,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/47d852db43c549afa13e26594a393b9d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/47d852db43c549afa13e26594a393b9d 2024-11-28T07:43:01,568 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/47d852db43c549afa13e26594a393b9d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T07:43:01,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/6bfd13e9adee4b53a786c13c2f8cc21c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/6bfd13e9adee4b53a786c13c2f8cc21c 2024-11-28T07:43:01,586 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/6bfd13e9adee4b53a786c13c2f8cc21c, entries=150, sequenceid=209, filesize=11.9 K 2024-11-28T07:43:01,587 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a71415a0af8a2acbd268f1bf9a3ef418 in 1311ms, sequenceid=209, compaction requested=false 2024-11-28T07:43:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-28T07:43:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-28T07:43:01,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-28T07:43:01,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4650 sec 2024-11-28T07:43:01,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4700 sec 2024-11-28T07:43:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:01,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:43:01,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:01,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:01,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:01,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:01,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:01,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:01,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/424665b8d02b4bdc85acbecb7bf0a34a is 50, key is test_row_0/A:col10/1732779780618/Put/seqid=0 2024-11-28T07:43:01,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742327_1503 (size=14541) 2024-11-28T07:43:01,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779841843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779841852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:01,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779841960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:01,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779841960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:01,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38268 deadline: 1732779842010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:02,020 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38224 deadline: 1732779842022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:02,027 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38210 deadline: 1732779842040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:02,047 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_1' in a71415a0af8a2acbd268f1bf9a3ef418 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:43:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779842170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779842180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/424665b8d02b4bdc85acbecb7bf0a34a 2024-11-28T07:43:02,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/08b0286735b248c284e631f70b65115a is 50, key is test_row_0/B:col10/1732779780618/Put/seqid=0 2024-11-28T07:43:02,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742328_1504 (size=12151) 2024-11-28T07:43:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-28T07:43:02,272 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:02,272 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-28T07:43:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T07:43:02,275 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:02,276 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:02,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T07:43:02,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T07:43:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:02,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779842481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:02,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779842500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:02,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:02,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:02,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T07:43:02,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:02,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:02,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T07:43:02,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/08b0286735b248c284e631f70b65115a 2024-11-28T07:43:02,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/48354cdf7423416eab830e7a1fcf5e4c is 50, key is test_row_0/C:col10/1732779780618/Put/seqid=0 2024-11-28T07:43:02,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742329_1505 (size=12151) 2024-11-28T07:43:02,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/48354cdf7423416eab830e7a1fcf5e4c 2024-11-28T07:43:02,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/424665b8d02b4bdc85acbecb7bf0a34a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/424665b8d02b4bdc85acbecb7bf0a34a 2024-11-28T07:43:02,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/424665b8d02b4bdc85acbecb7bf0a34a, entries=200, sequenceid=227, filesize=14.2 K 2024-11-28T07:43:02,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/08b0286735b248c284e631f70b65115a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/08b0286735b248c284e631f70b65115a 2024-11-28T07:43:02,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/08b0286735b248c284e631f70b65115a, entries=150, sequenceid=227, filesize=11.9 K 2024-11-28T07:43:02,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/48354cdf7423416eab830e7a1fcf5e4c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/48354cdf7423416eab830e7a1fcf5e4c 2024-11-28T07:43:02,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/48354cdf7423416eab830e7a1fcf5e4c, entries=150, sequenceid=227, filesize=11.9 K 2024-11-28T07:43:02,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a71415a0af8a2acbd268f1bf9a3ef418 in 866ms, sequenceid=227, compaction requested=true 2024-11-28T07:43:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:02,650 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:02,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:02,650 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:02,651 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:02,652 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:02,652 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:43:02,652 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:43:02,652 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,652 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,652 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/5c846b788de144e1a3bad1e726d1dea8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2b73ae9b4b6461591c3b8d47e0cd8f8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/424665b8d02b4bdc85acbecb7bf0a34a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=38.4 K 2024-11-28T07:43:02,652 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0bbc41034f274567a638912473ff0ce9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/47d852db43c549afa13e26594a393b9d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/08b0286735b248c284e631f70b65115a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.1 K 2024-11-28T07:43:02,652 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c846b788de144e1a3bad1e726d1dea8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779777177 2024-11-28T07:43:02,653 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bbc41034f274567a638912473ff0ce9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779777177 2024-11-28T07:43:02,657 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d852db43c549afa13e26594a393b9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732779778353 2024-11-28T07:43:02,657 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2b73ae9b4b6461591c3b8d47e0cd8f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732779778353 2024-11-28T07:43:02,657 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b0286735b248c284e631f70b65115a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779780583 2024-11-28T07:43:02,657 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424665b8d02b4bdc85acbecb7bf0a34a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779780583 2024-11-28T07:43:02,666 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:02,666 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:02,666 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e2a9e2112e814e4fb03630326c118802 is 50, key is test_row_0/B:col10/1732779780618/Put/seqid=0 2024-11-28T07:43:02,667 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/16238da98e354d00a3e3848db4a4934e is 50, key is test_row_0/A:col10/1732779780618/Put/seqid=0 2024-11-28T07:43:02,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742331_1507 (size=12765) 2024-11-28T07:43:02,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742330_1506 (size=12765) 2024-11-28T07:43:02,685 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/16238da98e354d00a3e3848db4a4934e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/16238da98e354d00a3e3848db4a4934e 2024-11-28T07:43:02,690 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 16238da98e354d00a3e3848db4a4934e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:02,690 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:02,690 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779782650; duration=0sec 2024-11-28T07:43:02,690 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:02,690 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:43:02,690 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:02,691 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:02,691 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:43:02,691 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,691 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/743f594ad0ab425db590a1bcd36ec5e4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/6bfd13e9adee4b53a786c13c2f8cc21c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/48354cdf7423416eab830e7a1fcf5e4c] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.1 K 2024-11-28T07:43:02,692 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 743f594ad0ab425db590a1bcd36ec5e4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779777177 2024-11-28T07:43:02,692 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bfd13e9adee4b53a786c13c2f8cc21c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732779778353 2024-11-28T07:43:02,692 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48354cdf7423416eab830e7a1fcf5e4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779780583 2024-11-28T07:43:02,703 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:02,703 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/ff148c4c218c4613afb59b2c142c0820 is 50, key is test_row_0/C:col10/1732779780618/Put/seqid=0 2024-11-28T07:43:02,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742332_1508 (size=12765) 2024-11-28T07:43:02,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-28T07:43:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:02,737 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:43:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:02,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:02,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/1545d3d77b0449a89d024e888061e184 is 50, key is test_row_0/A:col10/1732779781798/Put/seqid=0 2024-11-28T07:43:02,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742333_1509 (size=12151) 2024-11-28T07:43:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T07:43:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:03,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:03,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/e2a9e2112e814e4fb03630326c118802 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e2a9e2112e814e4fb03630326c118802 2024-11-28T07:43:03,092 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into e2a9e2112e814e4fb03630326c118802(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:03,092 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:03,092 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779782650; duration=0sec 2024-11-28T07:43:03,092 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:03,092 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:43:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779843089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779843093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,118 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/ff148c4c218c4613afb59b2c142c0820 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ff148c4c218c4613afb59b2c142c0820 2024-11-28T07:43:03,121 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into ff148c4c218c4613afb59b2c142c0820(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:03,122 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:03,122 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779782650; duration=0sec 2024-11-28T07:43:03,122 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:03,122 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:43:03,166 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/1545d3d77b0449a89d024e888061e184 2024-11-28T07:43:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/3949c7af4177400b8438890e882d70fa is 50, key is test_row_0/B:col10/1732779781798/Put/seqid=0 2024-11-28T07:43:03,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742334_1510 (size=12151) 2024-11-28T07:43:03,181 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/3949c7af4177400b8438890e882d70fa 2024-11-28T07:43:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/f8719e0a980340df99dd7888936dd196 is 50, key is test_row_0/C:col10/1732779781798/Put/seqid=0 2024-11-28T07:43:03,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742335_1511 (size=12151) 2024-11-28T07:43:03,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779843210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779843210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T07:43:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779843430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779843431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,594 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T07:43:03,605 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/f8719e0a980340df99dd7888936dd196 2024-11-28T07:43:03,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/1545d3d77b0449a89d024e888061e184 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1545d3d77b0449a89d024e888061e184 2024-11-28T07:43:03,612 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1545d3d77b0449a89d024e888061e184, entries=150, sequenceid=242, filesize=11.9 K 2024-11-28T07:43:03,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/3949c7af4177400b8438890e882d70fa as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/3949c7af4177400b8438890e882d70fa 2024-11-28T07:43:03,616 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/3949c7af4177400b8438890e882d70fa, entries=150, sequenceid=242, filesize=11.9 K 2024-11-28T07:43:03,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/f8719e0a980340df99dd7888936dd196 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f8719e0a980340df99dd7888936dd196 2024-11-28T07:43:03,621 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f8719e0a980340df99dd7888936dd196, entries=150, sequenceid=242, filesize=11.9 K 2024-11-28T07:43:03,622 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a71415a0af8a2acbd268f1bf9a3ef418 in 885ms, sequenceid=242, compaction requested=false 2024-11-28T07:43:03,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:03,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:03,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-28T07:43:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-28T07:43:03,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-28T07:43:03,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3470 sec 2024-11-28T07:43:03,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.3540 sec 2024-11-28T07:43:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:03,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-28T07:43:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:03,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:03,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/8a4381c711344ebb8fbdade8ff6a6810 is 50, key is test_row_0/A:col10/1732779783090/Put/seqid=0 2024-11-28T07:43:03,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742336_1512 (size=14691) 2024-11-28T07:43:03,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779843807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779843808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779843920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779843921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:04,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779844130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779844130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:04,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/8a4381c711344ebb8fbdade8ff6a6810 2024-11-28T07:43:04,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/891b1a8fcaf84155a1d25c42c5b88bad is 50, key is test_row_0/B:col10/1732779783090/Put/seqid=0 2024-11-28T07:43:04,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742337_1513 (size=12251) 2024-11-28T07:43:04,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/891b1a8fcaf84155a1d25c42c5b88bad 2024-11-28T07:43:04,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/0e89862f13ae4f6eabb671229259a594 is 50, key is test_row_0/C:col10/1732779783090/Put/seqid=0 2024-11-28T07:43:04,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742338_1514 (size=12251) 2024-11-28T07:43:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-28T07:43:04,420 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:04,421 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-28T07:43:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T07:43:04,423 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:04,424 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:04,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:04,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779844440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:04,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779844440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:04,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T07:43:04,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T07:43:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:04,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:04,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/0e89862f13ae4f6eabb671229259a594 2024-11-28T07:43:04,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/8a4381c711344ebb8fbdade8ff6a6810 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/8a4381c711344ebb8fbdade8ff6a6810 2024-11-28T07:43:04,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/8a4381c711344ebb8fbdade8ff6a6810, entries=200, sequenceid=263, filesize=14.3 K 2024-11-28T07:43:04,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/891b1a8fcaf84155a1d25c42c5b88bad as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/891b1a8fcaf84155a1d25c42c5b88bad 2024-11-28T07:43:04,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/891b1a8fcaf84155a1d25c42c5b88bad, entries=150, sequenceid=263, filesize=12.0 K 2024-11-28T07:43:04,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/0e89862f13ae4f6eabb671229259a594 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0e89862f13ae4f6eabb671229259a594 2024-11-28T07:43:04,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0e89862f13ae4f6eabb671229259a594, entries=150, sequenceid=263, filesize=12.0 K 2024-11-28T07:43:04,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a71415a0af8a2acbd268f1bf9a3ef418 in 887ms, sequenceid=263, compaction requested=true 2024-11-28T07:43:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:04,633 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:04,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:04,633 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:04,635 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39607 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:04,635 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:04,635 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:43:04,635 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:04,635 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:43:04,635 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/16238da98e354d00a3e3848db4a4934e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1545d3d77b0449a89d024e888061e184, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/8a4381c711344ebb8fbdade8ff6a6810] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=38.7 K 2024-11-28T07:43:04,635 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:04,635 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e2a9e2112e814e4fb03630326c118802, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/3949c7af4177400b8438890e882d70fa, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/891b1a8fcaf84155a1d25c42c5b88bad] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.3 K 2024-11-28T07:43:04,635 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16238da98e354d00a3e3848db4a4934e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779780583 2024-11-28T07:43:04,636 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting e2a9e2112e814e4fb03630326c118802, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779780583 2024-11-28T07:43:04,636 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1545d3d77b0449a89d024e888061e184, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732779781798 2024-11-28T07:43:04,636 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4381c711344ebb8fbdade8ff6a6810, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732779783072 2024-11-28T07:43:04,636 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 3949c7af4177400b8438890e882d70fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732779781798 2024-11-28T07:43:04,636 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 891b1a8fcaf84155a1d25c42c5b88bad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732779783072 2024-11-28T07:43:04,643 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:04,643 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:04,643 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/d4dac09bed6b45488dfae7e0335df22c is 50, key is test_row_0/B:col10/1732779783090/Put/seqid=0 2024-11-28T07:43:04,643 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/55e323c7837c41c382a7c3f382bf87ee is 50, key is test_row_0/A:col10/1732779783090/Put/seqid=0 2024-11-28T07:43:04,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742339_1515 (size=12967) 2024-11-28T07:43:04,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742340_1516 (size=12967) 2024-11-28T07:43:04,662 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/d4dac09bed6b45488dfae7e0335df22c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d4dac09bed6b45488dfae7e0335df22c 2024-11-28T07:43:04,662 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/55e323c7837c41c382a7c3f382bf87ee as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/55e323c7837c41c382a7c3f382bf87ee 2024-11-28T07:43:04,668 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into d4dac09bed6b45488dfae7e0335df22c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:04,668 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:04,668 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779784633; duration=0sec 2024-11-28T07:43:04,668 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:04,668 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:43:04,668 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:04,671 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37167 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:04,671 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:43:04,671 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:04,671 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ff148c4c218c4613afb59b2c142c0820, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f8719e0a980340df99dd7888936dd196, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0e89862f13ae4f6eabb671229259a594] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.3 K 2024-11-28T07:43:04,672 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 55e323c7837c41c382a7c3f382bf87ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:04,672 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:04,672 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779784633; duration=0sec 2024-11-28T07:43:04,672 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:04,672 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:43:04,672 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting ff148c4c218c4613afb59b2c142c0820, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732779780583 2024-11-28T07:43:04,673 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f8719e0a980340df99dd7888936dd196, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732779781798 2024-11-28T07:43:04,676 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e89862f13ae4f6eabb671229259a594, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732779783072 2024-11-28T07:43:04,695 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:04,695 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/5a9cffc463764a98807b2c2e6465a736 is 50, key is test_row_0/C:col10/1732779783090/Put/seqid=0 2024-11-28T07:43:04,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-28T07:43:04,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:04,729 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-28T07:43:04,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:04,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:04,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:04,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:04,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:04,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742341_1517 (size=12967) 2024-11-28T07:43:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T07:43:04,742 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/5a9cffc463764a98807b2c2e6465a736 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5a9cffc463764a98807b2c2e6465a736 2024-11-28T07:43:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/33918b1d60b54327a85a4bce6c9415de is 50, key is test_row_0/A:col10/1732779783755/Put/seqid=0 2024-11-28T07:43:04,750 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into 5a9cffc463764a98807b2c2e6465a736(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:04,750 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:04,750 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779784633; duration=0sec 2024-11-28T07:43:04,750 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:04,750 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:43:04,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742342_1518 (size=12301) 2024-11-28T07:43:04,787 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/33918b1d60b54327a85a4bce6c9415de 2024-11-28T07:43:04,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/2d4a83b6ed604137ba549cf852c01a7d is 50, key is test_row_0/B:col10/1732779783755/Put/seqid=0 2024-11-28T07:43:04,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742343_1519 (size=12301) 2024-11-28T07:43:04,830 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/2d4a83b6ed604137ba549cf852c01a7d 2024-11-28T07:43:04,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/ef84b57317964b5b8e7a770ac6c004ad is 50, key is test_row_0/C:col10/1732779783755/Put/seqid=0 2024-11-28T07:43:04,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742344_1520 (size=12301) 2024-11-28T07:43:04,859 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/ef84b57317964b5b8e7a770ac6c004ad 2024-11-28T07:43:04,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/33918b1d60b54327a85a4bce6c9415de as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/33918b1d60b54327a85a4bce6c9415de 2024-11-28T07:43:04,869 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/33918b1d60b54327a85a4bce6c9415de, entries=150, sequenceid=278, filesize=12.0 K 2024-11-28T07:43:04,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/2d4a83b6ed604137ba549cf852c01a7d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2d4a83b6ed604137ba549cf852c01a7d 2024-11-28T07:43:04,880 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2d4a83b6ed604137ba549cf852c01a7d, entries=150, sequenceid=278, filesize=12.0 K 2024-11-28T07:43:04,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/ef84b57317964b5b8e7a770ac6c004ad as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ef84b57317964b5b8e7a770ac6c004ad 2024-11-28T07:43:04,891 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ef84b57317964b5b8e7a770ac6c004ad, entries=150, sequenceid=278, filesize=12.0 K 2024-11-28T07:43:04,892 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for a71415a0af8a2acbd268f1bf9a3ef418 in 163ms, sequenceid=278, compaction requested=false 2024-11-28T07:43:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-28T07:43:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-28T07:43:04,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-28T07:43:04,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 470 msec 2024-11-28T07:43:04,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 478 msec 2024-11-28T07:43:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:04,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:43:04,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:04,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:04,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:04,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:04,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:04,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:04,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/3434d11ca00b4c16b2eba9f4221097ca is 50, key is test_row_0/A:col10/1732779784976/Put/seqid=0 2024-11-28T07:43:05,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742345_1521 (size=17181) 2024-11-28T07:43:05,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/3434d11ca00b4c16b2eba9f4221097ca 2024-11-28T07:43:05,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/153ffcce08454b10b5825f39741ef6c9 is 50, key is test_row_0/B:col10/1732779784976/Put/seqid=0 2024-11-28T07:43:05,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742346_1522 (size=12301) 2024-11-28T07:43:05,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/153ffcce08454b10b5825f39741ef6c9 2024-11-28T07:43:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-28T07:43:05,052 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:05,053 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:05,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-28T07:43:05,055 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T07:43:05,056 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:05,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:05,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/8b39098ab93c4cc38b90e727a00e325e is 50, key is test_row_0/C:col10/1732779784976/Put/seqid=0 2024-11-28T07:43:05,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779784990, max timestamp=1732779785098Num uniques 859; 2024-11-28T07:43:05,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:43:05,103 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779785066, max timestamp=1732779785098Num uniques 859; 2024-11-28T07:43:05,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:43:05,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779785066, max timestamp=1732779785098Num uniques 859; 2024-11-28T07:43:05,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:43:05,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742347_1523 (size=12301) 2024-11-28T07:43:05,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/8b39098ab93c4cc38b90e727a00e325e 2024-11-28T07:43:05,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/3434d11ca00b4c16b2eba9f4221097ca as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/3434d11ca00b4c16b2eba9f4221097ca 2024-11-28T07:43:05,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/3434d11ca00b4c16b2eba9f4221097ca, entries=250, sequenceid=290, filesize=16.8 K 2024-11-28T07:43:05,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/153ffcce08454b10b5825f39741ef6c9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/153ffcce08454b10b5825f39741ef6c9 2024-11-28T07:43:05,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/153ffcce08454b10b5825f39741ef6c9, entries=150, sequenceid=290, filesize=12.0 K 2024-11-28T07:43:05,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/8b39098ab93c4cc38b90e727a00e325e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/8b39098ab93c4cc38b90e727a00e325e 2024-11-28T07:43:05,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/8b39098ab93c4cc38b90e727a00e325e, entries=150, sequenceid=290, filesize=12.0 K 2024-11-28T07:43:05,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=49.20 KB/50380 for a71415a0af8a2acbd268f1bf9a3ef418 in 162ms, sequenceid=290, compaction requested=true 2024-11-28T07:43:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:05,141 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:05,141 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:05,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:05,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:05,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:05,143 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:05,143 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:43:05,143 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,143 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/55e323c7837c41c382a7c3f382bf87ee, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/33918b1d60b54327a85a4bce6c9415de, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/3434d11ca00b4c16b2eba9f4221097ca] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=41.5 K 2024-11-28T07:43:05,143 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:05,143 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:43:05,143 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,143 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d4dac09bed6b45488dfae7e0335df22c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2d4a83b6ed604137ba549cf852c01a7d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/153ffcce08454b10b5825f39741ef6c9] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.7 K 2024-11-28T07:43:05,144 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d4dac09bed6b45488dfae7e0335df22c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732779783072 2024-11-28T07:43:05,144 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55e323c7837c41c382a7c3f382bf87ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732779783072 2024-11-28T07:43:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:05,144 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d4a83b6ed604137ba549cf852c01a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732779783755 2024-11-28T07:43:05,144 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33918b1d60b54327a85a4bce6c9415de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732779783755 2024-11-28T07:43:05,145 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 153ffcce08454b10b5825f39741ef6c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779784966 2024-11-28T07:43:05,145 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3434d11ca00b4c16b2eba9f4221097ca, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779784955 2024-11-28T07:43:05,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=55.91 KB heapSize=151 KB 2024-11-28T07:43:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:05,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:05,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:05,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:05,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:05,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:05,159 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:05,159 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/82edf19859504ece9c35f375e56c1a9e is 50, key is test_row_0/A:col10/1732779784976/Put/seqid=0 2024-11-28T07:43:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T07:43:05,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/22be91623866487caba27db69edf3220 is 50, key is test_row_0/A:col10/1732779785139/Put/seqid=0 2024-11-28T07:43:05,166 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:05,167 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/12fd4f9ea8f54061aef805541f7c1690 is 50, key is test_row_0/B:col10/1732779784976/Put/seqid=0 2024-11-28T07:43:05,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742348_1524 (size=13119) 2024-11-28T07:43:05,200 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/82edf19859504ece9c35f375e56c1a9e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/82edf19859504ece9c35f375e56c1a9e 2024-11-28T07:43:05,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,214 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 82edf19859504ece9c35f375e56c1a9e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:05,215 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:05,215 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779785141; duration=0sec 2024-11-28T07:43:05,215 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:05,215 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:43:05,215 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:05,216 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:05,216 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:43:05,216 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,216 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5a9cffc463764a98807b2c2e6465a736, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ef84b57317964b5b8e7a770ac6c004ad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/8b39098ab93c4cc38b90e727a00e325e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.7 K 2024-11-28T07:43:05,217 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a9cffc463764a98807b2c2e6465a736, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1732779783072 2024-11-28T07:43:05,217 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef84b57317964b5b8e7a770ac6c004ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732779783755 2024-11-28T07:43:05,218 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b39098ab93c4cc38b90e727a00e325e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779784966 2024-11-28T07:43:05,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742349_1525 (size=14741) 2024-11-28T07:43:05,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742350_1526 (size=13119) 2024-11-28T07:43:05,244 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:05,244 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/38b70b0acfc4477bbc2eb82d31ccc8d5 is 50, key is test_row_0/C:col10/1732779784976/Put/seqid=0 2024-11-28T07:43:05,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742351_1527 (size=13119) 2024-11-28T07:43:05,281 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/38b70b0acfc4477bbc2eb82d31ccc8d5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/38b70b0acfc4477bbc2eb82d31ccc8d5 2024-11-28T07:43:05,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779785254, max timestamp=1732779785280Num uniques 859; 2024-11-28T07:43:05,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779785254, max timestamp=1732779785280Num uniques 859; 2024-11-28T07:43:05,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:43:05,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:43:05,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779785254, max timestamp=1732779785280Num uniques 859; 2024-11-28T07:43:05,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:43:05,288 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into 38b70b0acfc4477bbc2eb82d31ccc8d5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:05,288 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:05,288 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779785142; duration=0sec 2024-11-28T07:43:05,288 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:05,288 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:43:05,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T07:43:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779845366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779845371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:05,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779845490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779845490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:05,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/22be91623866487caba27db69edf3220 2024-11-28T07:43:05,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/06344df92b22466bb4ba85ed744492e9 is 50, key is test_row_0/B:col10/1732779785139/Put/seqid=0 2024-11-28T07:43:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742352_1528 (size=12301) 2024-11-28T07:43:05,637 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/12fd4f9ea8f54061aef805541f7c1690 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/12fd4f9ea8f54061aef805541f7c1690 2024-11-28T07:43:05,640 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into 12fd4f9ea8f54061aef805541f7c1690(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:05,640 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:05,640 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779785141; duration=0sec 2024-11-28T07:43:05,640 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:05,640 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:43:05,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T07:43:05,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779845710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:05,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,719 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779845710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:05,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:05,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:05,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:05,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:05,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/06344df92b22466bb4ba85ed744492e9 2024-11-28T07:43:06,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/bbbd6c5163a24485b6e698609332d0b8 is 50, key is test_row_0/C:col10/1732779785139/Put/seqid=0 2024-11-28T07:43:06,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742353_1529 (size=12301) 2024-11-28T07:43:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38234 deadline: 1732779846030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38250 deadline: 1732779846030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 , the old value is region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=2 because the exception is null or not the one we care about 2024-11-28T07:43:06,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T07:43:06,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,348 INFO [Thread-2039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2039. 2024-11-28T07:43:06,348 DEBUG [Thread-2039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:06,348 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,348 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:06,355 INFO [Thread-2045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2045. 2024-11-28T07:43:06,356 DEBUG [Thread-2045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:06,356 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,356 DEBUG [Thread-2045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:06,358 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-11-28T07:43:06,358 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:06,358 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,358 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,358 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:06,361 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-11-28T07:43:06,361 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:06,361 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,361 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:06,366 INFO [Thread-2037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2037. 2024-11-28T07:43:06,366 DEBUG [Thread-2037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:06,366 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,366 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:06,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/bbbd6c5163a24485b6e698609332d0b8 2024-11-28T07:43:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/22be91623866487caba27db69edf3220 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/22be91623866487caba27db69edf3220 2024-11-28T07:43:06,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/22be91623866487caba27db69edf3220, entries=200, sequenceid=315, filesize=14.4 K 2024-11-28T07:43:06,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/06344df92b22466bb4ba85ed744492e9 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/06344df92b22466bb4ba85ed744492e9 2024-11-28T07:43:06,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/06344df92b22466bb4ba85ed744492e9, entries=150, sequenceid=315, filesize=12.0 K 2024-11-28T07:43:06,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/bbbd6c5163a24485b6e698609332d0b8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/bbbd6c5163a24485b6e698609332d0b8 2024-11-28T07:43:06,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/bbbd6c5163a24485b6e698609332d0b8, entries=150, sequenceid=315, filesize=12.0 K 2024-11-28T07:43:06,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~55.91 KB/57250, heapSize ~150.95 KB/154576, currentSize=127.47 KB/130530 for a71415a0af8a2acbd268f1bf9a3ef418 in 1309ms, sequenceid=315, compaction requested=false 2024-11-28T07:43:06,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:06,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-28T07:43:06,552 INFO [Thread-2030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2030. 2024-11-28T07:43:06,552 INFO [Thread-2034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2034. 2024-11-28T07:43:06,552 DEBUG [Thread-2030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:06,552 DEBUG [Thread-2034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:06,552 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,552 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,552 DEBUG [Thread-2030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,552 DEBUG [Thread-2034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:06,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:06,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T07:43:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T07:43:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:06,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-28T07:43:06,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/98a28960862f42dab0aa0ec738b26b7b is 50, key is test_row_0/A:col10/1732779785370/Put/seqid=0 2024-11-28T07:43:06,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742354_1530 (size=12301) 2024-11-28T07:43:06,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:06,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:06,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:06,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:06,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:06,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/98a28960862f42dab0aa0ec738b26b7b 2024-11-28T07:43:06,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/9aa509368a46468087d573411bf007c4 is 50, key is test_row_0/B:col10/1732779785370/Put/seqid=0 2024-11-28T07:43:06,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742355_1531 (size=12301) 2024-11-28T07:43:07,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:07,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:07,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:07,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:07,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T07:43:07,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:07,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:07,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/9aa509368a46468087d573411bf007c4 2024-11-28T07:43:07,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/2a2cd399b6b2407dac4507efbf7ef988 is 50, key is test_row_0/C:col10/1732779785370/Put/seqid=0 2024-11-28T07:43:07,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742356_1532 (size=12301) 2024-11-28T07:43:07,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. as already flushing 2024-11-28T07:43:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:07,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/2a2cd399b6b2407dac4507efbf7ef988 2024-11-28T07:43:07,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/98a28960862f42dab0aa0ec738b26b7b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/98a28960862f42dab0aa0ec738b26b7b 2024-11-28T07:43:07,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/98a28960862f42dab0aa0ec738b26b7b, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T07:43:07,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/9aa509368a46468087d573411bf007c4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9aa509368a46468087d573411bf007c4 2024-11-28T07:43:07,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9aa509368a46468087d573411bf007c4, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T07:43:07,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/2a2cd399b6b2407dac4507efbf7ef988 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/2a2cd399b6b2407dac4507efbf7ef988 2024-11-28T07:43:07,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/2a2cd399b6b2407dac4507efbf7ef988, entries=150, sequenceid=356, filesize=12.0 K 2024-11-28T07:43:07,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=0 B/0 for a71415a0af8a2acbd268f1bf9a3ef418 in 1236ms, sequenceid=356, compaction requested=true 2024-11-28T07:43:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a71415a0af8a2acbd268f1bf9a3ef418:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:07,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:07,789 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:07,789 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:07,789 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:07,789 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:07,789 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/A is initiating minor compaction (all files) 2024-11-28T07:43:07,789 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/B is initiating minor compaction (all files) 2024-11-28T07:43:07,790 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/A in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,790 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/B in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,790 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/12fd4f9ea8f54061aef805541f7c1690, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/06344df92b22466bb4ba85ed744492e9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9aa509368a46468087d573411bf007c4] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.8 K 2024-11-28T07:43:07,790 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/82edf19859504ece9c35f375e56c1a9e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/22be91623866487caba27db69edf3220, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/98a28960862f42dab0aa0ec738b26b7b] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=39.2 K 2024-11-28T07:43:07,790 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82edf19859504ece9c35f375e56c1a9e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779784966 2024-11-28T07:43:07,790 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 12fd4f9ea8f54061aef805541f7c1690, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779784966 2024-11-28T07:43:07,790 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 06344df92b22466bb4ba85ed744492e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732779785083 2024-11-28T07:43:07,790 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22be91623866487caba27db69edf3220, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732779785083 2024-11-28T07:43:07,790 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aa509368a46468087d573411bf007c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732779785329 2024-11-28T07:43:07,790 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a28960862f42dab0aa0ec738b26b7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732779785329 2024-11-28T07:43:07,796 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:07,796 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#A#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:07,797 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0761a7994c404f3f94e04ca8bced4373 is 50, key is test_row_0/B:col10/1732779785370/Put/seqid=0 2024-11-28T07:43:07,797 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/73c78b1eca264403bb098c611cd58529 is 50, key is test_row_0/A:col10/1732779785370/Put/seqid=0 2024-11-28T07:43:07,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742357_1533 (size=13221) 2024-11-28T07:43:07,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742358_1534 (size=13221) 2024-11-28T07:43:07,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-28T07:43:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-28T07:43:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-28T07:43:07,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-28T07:43:07,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7560 sec 2024-11-28T07:43:07,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.7610 sec 2024-11-28T07:43:08,203 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/0761a7994c404f3f94e04ca8bced4373 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0761a7994c404f3f94e04ca8bced4373 2024-11-28T07:43:08,203 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/73c78b1eca264403bb098c611cd58529 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/73c78b1eca264403bb098c611cd58529 2024-11-28T07:43:08,207 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/B of a71415a0af8a2acbd268f1bf9a3ef418 into 0761a7994c404f3f94e04ca8bced4373(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:08,207 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/A of a71415a0af8a2acbd268f1bf9a3ef418 into 73c78b1eca264403bb098c611cd58529(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:08,207 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:08,207 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:08,207 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/A, priority=13, startTime=1732779787789; duration=0sec 2024-11-28T07:43:08,207 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/B, priority=13, startTime=1732779787789; duration=0sec 2024-11-28T07:43:08,207 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:08,207 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:08,207 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:B 2024-11-28T07:43:08,207 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:A 2024-11-28T07:43:08,207 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:08,208 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:08,208 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a71415a0af8a2acbd268f1bf9a3ef418/C is initiating minor compaction (all files) 2024-11-28T07:43:08,208 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a71415a0af8a2acbd268f1bf9a3ef418/C in TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:08,208 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/38b70b0acfc4477bbc2eb82d31ccc8d5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/bbbd6c5163a24485b6e698609332d0b8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/2a2cd399b6b2407dac4507efbf7ef988] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp, totalSize=36.8 K 2024-11-28T07:43:08,208 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b70b0acfc4477bbc2eb82d31ccc8d5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732779784966 2024-11-28T07:43:08,208 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting bbbd6c5163a24485b6e698609332d0b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732779785083 2024-11-28T07:43:08,209 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a2cd399b6b2407dac4507efbf7ef988, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732779785329 2024-11-28T07:43:08,216 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a71415a0af8a2acbd268f1bf9a3ef418#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:08,216 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/3772624e723f491482c0ab4add5321ff is 50, key is test_row_0/C:col10/1732779785370/Put/seqid=0 2024-11-28T07:43:08,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742359_1535 (size=13221) 2024-11-28T07:43:08,623 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/3772624e723f491482c0ab4add5321ff as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/3772624e723f491482c0ab4add5321ff 2024-11-28T07:43:08,627 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a71415a0af8a2acbd268f1bf9a3ef418/C of a71415a0af8a2acbd268f1bf9a3ef418 into 3772624e723f491482c0ab4add5321ff(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:08,627 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a71415a0af8a2acbd268f1bf9a3ef418: 2024-11-28T07:43:08,627 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., storeName=a71415a0af8a2acbd268f1bf9a3ef418/C, priority=13, startTime=1732779787789; duration=0sec 2024-11-28T07:43:08,627 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:08,627 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a71415a0af8a2acbd268f1bf9a3ef418:C 2024-11-28T07:43:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-28T07:43:09,221 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:12,061 INFO [Thread-2032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2032. 2024-11-28T07:43:12,061 DEBUG [Thread-2032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:12,062 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,062 DEBUG [Thread-2032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:12,092 INFO [Thread-2028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2028. 2024-11-28T07:43:12,092 DEBUG [Thread-2028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:12,092 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,092 DEBUG [Thread-2028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:12,131 INFO [Thread-2026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2026. 2024-11-28T07:43:12,131 DEBUG [Thread-2026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:12,131 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,131 DEBUG [Thread-2026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1274 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3811 rows 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1267 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3800 rows 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1271 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3808 rows 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1279 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3837 rows 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1272 2024-11-28T07:43:12,132 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3815 rows 2024-11-28T07:43:12,132 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:43:12,132 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:43:12,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,132 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:12,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:12,133 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T07:43:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T07:43:12,135 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779792135"}]},"ts":"1732779792135"} 2024-11-28T07:43:12,136 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T07:43:12,136 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T07:43:12,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:43:12,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, UNASSIGN}] 2024-11-28T07:43:12,138 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, UNASSIGN 2024-11-28T07:43:12,139 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a71415a0af8a2acbd268f1bf9a3ef418, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:12,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, UNASSIGN because future has completed 2024-11-28T07:43:12,140 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:43:12,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:43:12,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T07:43:12,292 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:12,292 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:43:12,292 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing a71415a0af8a2acbd268f1bf9a3ef418, disabling compactions & flushes 2024-11-28T07:43:12,292 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:12,292 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:12,292 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. after waiting 0 ms 2024-11-28T07:43:12,292 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:12,292 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing a71415a0af8a2acbd268f1bf9a3ef418 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-28T07:43:12,293 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=A 2024-11-28T07:43:12,293 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:12,293 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=B 2024-11-28T07:43:12,293 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:12,293 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a71415a0af8a2acbd268f1bf9a3ef418, store=C 2024-11-28T07:43:12,293 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:12,296 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/1f7e11c1234740599de9f7d68b19e050 is 50, key is test_row_1/A:col10/1732779792091/Put/seqid=0 2024-11-28T07:43:12,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742360_1536 (size=9857) 2024-11-28T07:43:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T07:43:12,699 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/1f7e11c1234740599de9f7d68b19e050 2024-11-28T07:43:12,705 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/b25c39c9b2d346688a98fc6dd5367e65 is 50, key is test_row_1/B:col10/1732779792091/Put/seqid=0 2024-11-28T07:43:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742361_1537 (size=9857) 2024-11-28T07:43:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T07:43:13,108 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/b25c39c9b2d346688a98fc6dd5367e65 2024-11-28T07:43:13,113 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/e6cb1474a9834e968e7cb85f06463916 is 50, key is test_row_1/C:col10/1732779792091/Put/seqid=0 2024-11-28T07:43:13,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742362_1538 (size=9857) 2024-11-28T07:43:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T07:43:13,517 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/e6cb1474a9834e968e7cb85f06463916 2024-11-28T07:43:13,520 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/A/1f7e11c1234740599de9f7d68b19e050 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1f7e11c1234740599de9f7d68b19e050 2024-11-28T07:43:13,523 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1f7e11c1234740599de9f7d68b19e050, entries=100, sequenceid=366, filesize=9.6 K 2024-11-28T07:43:13,523 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/B/b25c39c9b2d346688a98fc6dd5367e65 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b25c39c9b2d346688a98fc6dd5367e65 2024-11-28T07:43:13,525 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b25c39c9b2d346688a98fc6dd5367e65, entries=100, sequenceid=366, filesize=9.6 K 2024-11-28T07:43:13,526 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/.tmp/C/e6cb1474a9834e968e7cb85f06463916 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e6cb1474a9834e968e7cb85f06463916 2024-11-28T07:43:13,528 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e6cb1474a9834e968e7cb85f06463916, entries=100, sequenceid=366, filesize=9.6 K 2024-11-28T07:43:13,529 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a71415a0af8a2acbd268f1bf9a3ef418 in 1237ms, sequenceid=366, compaction requested=false 2024-11-28T07:43:13,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/23551abec22840bfb469eca8f4bfea6f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/2d1ebe989be945cc9e0a7839470c84f0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/393912ea1fbc4a5a8512f05a2bcf9622, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/db4262a0003b40498f8597b71d04c018, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/df40ed09c9954a6b9fc68c4e61ee6344, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88dbfbc747f340f0996562502b1f5bb7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/bf0e1f5a35ec484288d15c1cf8a805d3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/72287a34018241d4a8e89e2e76cc0513, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/fe263e87d6c64908a4fd3ea5647182a7, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88f1bb86975849bfb13637148ae8a46b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2494938155e4fe7a2be0f3df3b25622, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/7633bf6552ba48fb98d1ded91ec748fc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/15d301aa6d9e4ad4a744ab624b87c75f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e9674d59b9784c48bc85aaafc04982ea, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/5c846b788de144e1a3bad1e726d1dea8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/78f93030a7fe479baa35ed304b82f25d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2b73ae9b4b6461591c3b8d47e0cd8f8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/424665b8d02b4bdc85acbecb7bf0a34a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/16238da98e354d00a3e3848db4a4934e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1545d3d77b0449a89d024e888061e184, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/8a4381c711344ebb8fbdade8ff6a6810, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/55e323c7837c41c382a7c3f382bf87ee, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/33918b1d60b54327a85a4bce6c9415de, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/3434d11ca00b4c16b2eba9f4221097ca, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/82edf19859504ece9c35f375e56c1a9e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/22be91623866487caba27db69edf3220, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/98a28960862f42dab0aa0ec738b26b7b] to archive 2024-11-28T07:43:13,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:43:13,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/23551abec22840bfb469eca8f4bfea6f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/23551abec22840bfb469eca8f4bfea6f 2024-11-28T07:43:13,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/2d1ebe989be945cc9e0a7839470c84f0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/2d1ebe989be945cc9e0a7839470c84f0 2024-11-28T07:43:13,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/393912ea1fbc4a5a8512f05a2bcf9622 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/393912ea1fbc4a5a8512f05a2bcf9622 2024-11-28T07:43:13,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/db4262a0003b40498f8597b71d04c018 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/db4262a0003b40498f8597b71d04c018 2024-11-28T07:43:13,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/df40ed09c9954a6b9fc68c4e61ee6344 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/df40ed09c9954a6b9fc68c4e61ee6344 2024-11-28T07:43:13,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88dbfbc747f340f0996562502b1f5bb7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88dbfbc747f340f0996562502b1f5bb7 2024-11-28T07:43:13,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/bf0e1f5a35ec484288d15c1cf8a805d3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/bf0e1f5a35ec484288d15c1cf8a805d3 2024-11-28T07:43:13,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/72287a34018241d4a8e89e2e76cc0513 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/72287a34018241d4a8e89e2e76cc0513 2024-11-28T07:43:13,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/fe263e87d6c64908a4fd3ea5647182a7 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/fe263e87d6c64908a4fd3ea5647182a7 2024-11-28T07:43:13,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88f1bb86975849bfb13637148ae8a46b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/88f1bb86975849bfb13637148ae8a46b 2024-11-28T07:43:13,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2494938155e4fe7a2be0f3df3b25622 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2494938155e4fe7a2be0f3df3b25622 2024-11-28T07:43:13,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/7633bf6552ba48fb98d1ded91ec748fc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/7633bf6552ba48fb98d1ded91ec748fc 2024-11-28T07:43:13,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/15d301aa6d9e4ad4a744ab624b87c75f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/15d301aa6d9e4ad4a744ab624b87c75f 2024-11-28T07:43:13,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e9674d59b9784c48bc85aaafc04982ea to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e9674d59b9784c48bc85aaafc04982ea 2024-11-28T07:43:13,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/5c846b788de144e1a3bad1e726d1dea8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/5c846b788de144e1a3bad1e726d1dea8 2024-11-28T07:43:13,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/78f93030a7fe479baa35ed304b82f25d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/78f93030a7fe479baa35ed304b82f25d 2024-11-28T07:43:13,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2b73ae9b4b6461591c3b8d47e0cd8f8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/e2b73ae9b4b6461591c3b8d47e0cd8f8 2024-11-28T07:43:13,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/424665b8d02b4bdc85acbecb7bf0a34a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/424665b8d02b4bdc85acbecb7bf0a34a 2024-11-28T07:43:13,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/16238da98e354d00a3e3848db4a4934e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/16238da98e354d00a3e3848db4a4934e 2024-11-28T07:43:13,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1545d3d77b0449a89d024e888061e184 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1545d3d77b0449a89d024e888061e184 2024-11-28T07:43:13,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/8a4381c711344ebb8fbdade8ff6a6810 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/8a4381c711344ebb8fbdade8ff6a6810 2024-11-28T07:43:13,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/55e323c7837c41c382a7c3f382bf87ee to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/55e323c7837c41c382a7c3f382bf87ee 2024-11-28T07:43:13,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/33918b1d60b54327a85a4bce6c9415de to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/33918b1d60b54327a85a4bce6c9415de 2024-11-28T07:43:13,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/3434d11ca00b4c16b2eba9f4221097ca to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/3434d11ca00b4c16b2eba9f4221097ca 2024-11-28T07:43:13,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/82edf19859504ece9c35f375e56c1a9e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/82edf19859504ece9c35f375e56c1a9e 2024-11-28T07:43:13,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/22be91623866487caba27db69edf3220 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/22be91623866487caba27db69edf3220 2024-11-28T07:43:13,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/98a28960862f42dab0aa0ec738b26b7b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/98a28960862f42dab0aa0ec738b26b7b 2024-11-28T07:43:13,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/1392fb20025f48c8924219de586670ba, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/ea8fa49dd29e4f1a9f946a93955fcfad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/a41d2f7eee0847e88986221d895b363e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e06f1bb137ee45f5b02f99ab9fa5a202, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0dca7316985849619b47c9c7681dacf5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7619c7aecb7f4d049460d779322ffff0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d65b487c07d3487cb7f0b12fa331a6a1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/761d7af3b94c4dd1b78d416bbfdcd012, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7bf5447576b346b0b7face8960d40fad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e1df8bc6a3f04930b2880acdd342bb1f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0002089b1c2a462bb81a0dbd9992b0ce, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9a6e529c7a72440f8268635675ee74ed, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/24ad87e141c84f32b150c8736d5dcd3a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2aee833bae1f4be5b7974a71eb833868, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0bbc41034f274567a638912473ff0ce9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b149b5e17ae84a1d8a8a09645a0ae626, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/47d852db43c549afa13e26594a393b9d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e2a9e2112e814e4fb03630326c118802, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/08b0286735b248c284e631f70b65115a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/3949c7af4177400b8438890e882d70fa, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d4dac09bed6b45488dfae7e0335df22c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/891b1a8fcaf84155a1d25c42c5b88bad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2d4a83b6ed604137ba549cf852c01a7d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/12fd4f9ea8f54061aef805541f7c1690, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/153ffcce08454b10b5825f39741ef6c9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/06344df92b22466bb4ba85ed744492e9, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9aa509368a46468087d573411bf007c4] to archive 2024-11-28T07:43:13,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:43:13,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/1392fb20025f48c8924219de586670ba to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/1392fb20025f48c8924219de586670ba 2024-11-28T07:43:13,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/ea8fa49dd29e4f1a9f946a93955fcfad to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/ea8fa49dd29e4f1a9f946a93955fcfad 2024-11-28T07:43:13,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/a41d2f7eee0847e88986221d895b363e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/a41d2f7eee0847e88986221d895b363e 2024-11-28T07:43:13,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e06f1bb137ee45f5b02f99ab9fa5a202 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e06f1bb137ee45f5b02f99ab9fa5a202 2024-11-28T07:43:13,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0dca7316985849619b47c9c7681dacf5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0dca7316985849619b47c9c7681dacf5 2024-11-28T07:43:13,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7619c7aecb7f4d049460d779322ffff0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7619c7aecb7f4d049460d779322ffff0 2024-11-28T07:43:13,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d65b487c07d3487cb7f0b12fa331a6a1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d65b487c07d3487cb7f0b12fa331a6a1 2024-11-28T07:43:13,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/761d7af3b94c4dd1b78d416bbfdcd012 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/761d7af3b94c4dd1b78d416bbfdcd012 2024-11-28T07:43:13,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7bf5447576b346b0b7face8960d40fad to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/7bf5447576b346b0b7face8960d40fad 2024-11-28T07:43:13,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e1df8bc6a3f04930b2880acdd342bb1f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e1df8bc6a3f04930b2880acdd342bb1f 2024-11-28T07:43:13,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0002089b1c2a462bb81a0dbd9992b0ce to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0002089b1c2a462bb81a0dbd9992b0ce 2024-11-28T07:43:13,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9a6e529c7a72440f8268635675ee74ed to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9a6e529c7a72440f8268635675ee74ed 2024-11-28T07:43:13,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/24ad87e141c84f32b150c8736d5dcd3a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/24ad87e141c84f32b150c8736d5dcd3a 2024-11-28T07:43:13,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2aee833bae1f4be5b7974a71eb833868 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2aee833bae1f4be5b7974a71eb833868 2024-11-28T07:43:13,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0bbc41034f274567a638912473ff0ce9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0bbc41034f274567a638912473ff0ce9 2024-11-28T07:43:13,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b149b5e17ae84a1d8a8a09645a0ae626 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b149b5e17ae84a1d8a8a09645a0ae626 2024-11-28T07:43:13,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/47d852db43c549afa13e26594a393b9d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/47d852db43c549afa13e26594a393b9d 2024-11-28T07:43:13,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e2a9e2112e814e4fb03630326c118802 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/e2a9e2112e814e4fb03630326c118802 2024-11-28T07:43:13,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/08b0286735b248c284e631f70b65115a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/08b0286735b248c284e631f70b65115a 2024-11-28T07:43:13,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/3949c7af4177400b8438890e882d70fa to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/3949c7af4177400b8438890e882d70fa 2024-11-28T07:43:13,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d4dac09bed6b45488dfae7e0335df22c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/d4dac09bed6b45488dfae7e0335df22c 2024-11-28T07:43:13,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/891b1a8fcaf84155a1d25c42c5b88bad to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/891b1a8fcaf84155a1d25c42c5b88bad 2024-11-28T07:43:13,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2d4a83b6ed604137ba549cf852c01a7d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/2d4a83b6ed604137ba549cf852c01a7d 2024-11-28T07:43:13,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/12fd4f9ea8f54061aef805541f7c1690 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/12fd4f9ea8f54061aef805541f7c1690 2024-11-28T07:43:13,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/153ffcce08454b10b5825f39741ef6c9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/153ffcce08454b10b5825f39741ef6c9 2024-11-28T07:43:13,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/06344df92b22466bb4ba85ed744492e9 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/06344df92b22466bb4ba85ed744492e9 2024-11-28T07:43:13,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9aa509368a46468087d573411bf007c4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/9aa509368a46468087d573411bf007c4 2024-11-28T07:43:13,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5de9eaaa3252442e82cf0493e28e54d8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/81e1d6458ee44092b11f5d4878ac363a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0f48654adf0b4d22b9f4cc3ead061ef8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f2a7a15d51d34ce58559f25b1922d97c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/1919ba0fd5d24803828ae3a5e41a2b78, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/b59bec35264f4f5ab4ec1b3baba7b62f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/61f8985fafca4e37a577bfbf0742d58e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/08298e72c5194ce2b7c7cb9add0eb5f5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c4346dfe09644b74aa9ef03086b8e75f, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/a908083c0405487d8da34a4e507f6ecd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c049e0a97a3549c49b76521d87fcd829, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54c0ea4776324d4f9d8ccd801730d13c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/383e6c4e1ae645b2a6a951080203bd01, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54e9a03386c34b93b12e01055b957eaa, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/743f594ad0ab425db590a1bcd36ec5e4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e08f95a0953b4a23b241aff64d4db9ff, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/6bfd13e9adee4b53a786c13c2f8cc21c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ff148c4c218c4613afb59b2c142c0820, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/48354cdf7423416eab830e7a1fcf5e4c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f8719e0a980340df99dd7888936dd196, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5a9cffc463764a98807b2c2e6465a736, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0e89862f13ae4f6eabb671229259a594, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ef84b57317964b5b8e7a770ac6c004ad, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/38b70b0acfc4477bbc2eb82d31ccc8d5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/8b39098ab93c4cc38b90e727a00e325e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/bbbd6c5163a24485b6e698609332d0b8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/2a2cd399b6b2407dac4507efbf7ef988] to archive 2024-11-28T07:43:13,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:43:13,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5de9eaaa3252442e82cf0493e28e54d8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5de9eaaa3252442e82cf0493e28e54d8 2024-11-28T07:43:13,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/81e1d6458ee44092b11f5d4878ac363a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/81e1d6458ee44092b11f5d4878ac363a 2024-11-28T07:43:13,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0f48654adf0b4d22b9f4cc3ead061ef8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0f48654adf0b4d22b9f4cc3ead061ef8 2024-11-28T07:43:13,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f2a7a15d51d34ce58559f25b1922d97c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f2a7a15d51d34ce58559f25b1922d97c 2024-11-28T07:43:13,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/1919ba0fd5d24803828ae3a5e41a2b78 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/1919ba0fd5d24803828ae3a5e41a2b78 2024-11-28T07:43:13,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/b59bec35264f4f5ab4ec1b3baba7b62f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/b59bec35264f4f5ab4ec1b3baba7b62f 2024-11-28T07:43:13,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/61f8985fafca4e37a577bfbf0742d58e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/61f8985fafca4e37a577bfbf0742d58e 2024-11-28T07:43:13,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/08298e72c5194ce2b7c7cb9add0eb5f5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/08298e72c5194ce2b7c7cb9add0eb5f5 2024-11-28T07:43:13,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c4346dfe09644b74aa9ef03086b8e75f to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c4346dfe09644b74aa9ef03086b8e75f 2024-11-28T07:43:13,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/a908083c0405487d8da34a4e507f6ecd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/a908083c0405487d8da34a4e507f6ecd 2024-11-28T07:43:13,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c049e0a97a3549c49b76521d87fcd829 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/c049e0a97a3549c49b76521d87fcd829 2024-11-28T07:43:13,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54c0ea4776324d4f9d8ccd801730d13c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54c0ea4776324d4f9d8ccd801730d13c 2024-11-28T07:43:13,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/383e6c4e1ae645b2a6a951080203bd01 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/383e6c4e1ae645b2a6a951080203bd01 2024-11-28T07:43:13,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54e9a03386c34b93b12e01055b957eaa to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/54e9a03386c34b93b12e01055b957eaa 2024-11-28T07:43:13,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/743f594ad0ab425db590a1bcd36ec5e4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/743f594ad0ab425db590a1bcd36ec5e4 2024-11-28T07:43:13,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e08f95a0953b4a23b241aff64d4db9ff to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e08f95a0953b4a23b241aff64d4db9ff 2024-11-28T07:43:13,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/6bfd13e9adee4b53a786c13c2f8cc21c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/6bfd13e9adee4b53a786c13c2f8cc21c 2024-11-28T07:43:13,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ff148c4c218c4613afb59b2c142c0820 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ff148c4c218c4613afb59b2c142c0820 2024-11-28T07:43:13,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/48354cdf7423416eab830e7a1fcf5e4c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/48354cdf7423416eab830e7a1fcf5e4c 2024-11-28T07:43:13,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f8719e0a980340df99dd7888936dd196 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/f8719e0a980340df99dd7888936dd196 2024-11-28T07:43:13,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5a9cffc463764a98807b2c2e6465a736 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/5a9cffc463764a98807b2c2e6465a736 2024-11-28T07:43:13,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0e89862f13ae4f6eabb671229259a594 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/0e89862f13ae4f6eabb671229259a594 2024-11-28T07:43:13,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ef84b57317964b5b8e7a770ac6c004ad to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/ef84b57317964b5b8e7a770ac6c004ad 2024-11-28T07:43:13,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/38b70b0acfc4477bbc2eb82d31ccc8d5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/38b70b0acfc4477bbc2eb82d31ccc8d5 2024-11-28T07:43:13,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/8b39098ab93c4cc38b90e727a00e325e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/8b39098ab93c4cc38b90e727a00e325e 2024-11-28T07:43:13,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/bbbd6c5163a24485b6e698609332d0b8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/bbbd6c5163a24485b6e698609332d0b8 2024-11-28T07:43:13,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/2a2cd399b6b2407dac4507efbf7ef988 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/2a2cd399b6b2407dac4507efbf7ef988 2024-11-28T07:43:13,599 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/recovered.edits/369.seqid, newMaxSeqId=369, maxSeqId=1 2024-11-28T07:43:13,600 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. 2024-11-28T07:43:13,600 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for a71415a0af8a2acbd268f1bf9a3ef418: Waiting for close lock at 1732779792292Running coprocessor pre-close hooks at 1732779792292Disabling compacts and flushes for region at 1732779792292Disabling writes for close at 1732779792292Obtaining lock to block concurrent updates at 1732779792292Preparing flush snapshotting stores in a71415a0af8a2acbd268f1bf9a3ef418 at 1732779792292Finished memstore snapshotting TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732779792293 (+1 ms)Flushing stores of TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418. at 1732779792293Flushing a71415a0af8a2acbd268f1bf9a3ef418/A: creating writer at 1732779792293Flushing a71415a0af8a2acbd268f1bf9a3ef418/A: appending metadata at 1732779792295 (+2 ms)Flushing a71415a0af8a2acbd268f1bf9a3ef418/A: closing flushed file at 1732779792295Flushing a71415a0af8a2acbd268f1bf9a3ef418/B: creating writer at 1732779792702 (+407 ms)Flushing a71415a0af8a2acbd268f1bf9a3ef418/B: appending metadata at 1732779792704 (+2 ms)Flushing a71415a0af8a2acbd268f1bf9a3ef418/B: closing flushed file at 1732779792704Flushing a71415a0af8a2acbd268f1bf9a3ef418/C: creating writer at 1732779793111 (+407 ms)Flushing a71415a0af8a2acbd268f1bf9a3ef418/C: appending metadata at 1732779793113 (+2 ms)Flushing a71415a0af8a2acbd268f1bf9a3ef418/C: closing flushed file at 1732779793113Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e2d0ff4: reopening flushed file at 1732779793520 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ad92b: reopening flushed file at 1732779793523 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35ddcacc: reopening flushed file at 1732779793526 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a71415a0af8a2acbd268f1bf9a3ef418 in 1237ms, sequenceid=366, compaction requested=false at 1732779793529 (+3 ms)Writing region close event to WAL at 1732779793597 (+68 ms)Running coprocessor post-close hooks at 1732779793600 (+3 ms)Closed at 1732779793600 2024-11-28T07:43:13,601 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:13,602 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=a71415a0af8a2acbd268f1bf9a3ef418, regionState=CLOSED 2024-11-28T07:43:13,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:43:13,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-28T07:43:13,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure a71415a0af8a2acbd268f1bf9a3ef418, server=f64cf8bdf19b,34437,1732779635619 in 1.4640 sec 2024-11-28T07:43:13,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-28T07:43:13,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a71415a0af8a2acbd268f1bf9a3ef418, UNASSIGN in 1.4670 sec 2024-11-28T07:43:13,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-28T07:43:13,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4700 sec 2024-11-28T07:43:13,609 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779793609"}]},"ts":"1732779793609"} 2024-11-28T07:43:13,611 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T07:43:13,611 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T07:43:13,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4780 sec 2024-11-28T07:43:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-28T07:43:14,281 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:14,281 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T07:43:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:14,283 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:14,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T07:43:14,283 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:14,286 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:14,287 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/recovered.edits] 2024-11-28T07:43:14,289 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1f7e11c1234740599de9f7d68b19e050 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/1f7e11c1234740599de9f7d68b19e050 2024-11-28T07:43:14,290 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/73c78b1eca264403bb098c611cd58529 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/A/73c78b1eca264403bb098c611cd58529 2024-11-28T07:43:14,291 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0761a7994c404f3f94e04ca8bced4373 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/0761a7994c404f3f94e04ca8bced4373 2024-11-28T07:43:14,292 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b25c39c9b2d346688a98fc6dd5367e65 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/B/b25c39c9b2d346688a98fc6dd5367e65 2024-11-28T07:43:14,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/3772624e723f491482c0ab4add5321ff to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/3772624e723f491482c0ab4add5321ff 2024-11-28T07:43:14,294 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e6cb1474a9834e968e7cb85f06463916 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/C/e6cb1474a9834e968e7cb85f06463916 2024-11-28T07:43:14,296 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/recovered.edits/369.seqid to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418/recovered.edits/369.seqid 2024-11-28T07:43:14,297 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a71415a0af8a2acbd268f1bf9a3ef418 2024-11-28T07:43:14,297 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T07:43:14,298 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:14,299 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T07:43:14,301 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T07:43:14,302 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:14,302 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T07:43:14,302 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732779794302"}]},"ts":"9223372036854775807"} 2024-11-28T07:43:14,303 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T07:43:14,303 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a71415a0af8a2acbd268f1bf9a3ef418, NAME => 'TestAcidGuarantees,,1732779764149.a71415a0af8a2acbd268f1bf9a3ef418.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T07:43:14,303 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T07:43:14,303 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732779794303"}]},"ts":"9223372036854775807"} 2024-11-28T07:43:14,304 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T07:43:14,305 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:14,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 24 msec 2024-11-28T07:43:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-28T07:43:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T07:43:14,390 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:14,401 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 248), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 341), ProcessCount=11 (was 11), AvailableMemoryMB=6196 (was 6292) 2024-11-28T07:43:14,410 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=6195 2024-11-28T07:43:14,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:43:14,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:43:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:14,413 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T07:43:14,413 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:14,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-28T07:43:14,414 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T07:43:14,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T07:43:14,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742363_1539 (size=1061) 2024-11-28T07:43:14,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T07:43:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T07:43:14,820 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573 2024-11-28T07:43:14,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742364_1540 (size=53) 2024-11-28T07:43:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T07:43:15,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T07:43:15,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:43:15,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a4c1644781f0b1f4c5e05c56d56502e2, disabling compactions & flushes 2024-11-28T07:43:15,226 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. after waiting 0 ms 2024-11-28T07:43:15,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,226 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,227 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a4c1644781f0b1f4c5e05c56d56502e2: Waiting for close lock at 1732779795226Disabling compacts and flushes for region at 1732779795226Disabling writes for close at 1732779795226Writing region close event to WAL at 1732779795226Closed at 1732779795226 2024-11-28T07:43:15,227 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T07:43:15,228 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732779795227"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732779795227"}]},"ts":"1732779795227"} 2024-11-28T07:43:15,229 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T07:43:15,230 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T07:43:15,230 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779795230"}]},"ts":"1732779795230"} 2024-11-28T07:43:15,231 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-28T07:43:15,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, ASSIGN}] 2024-11-28T07:43:15,232 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, ASSIGN 2024-11-28T07:43:15,232 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, ASSIGN; state=OFFLINE, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=false 2024-11-28T07:43:15,383 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:15,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, ASSIGN because future has completed 2024-11-28T07:43:15,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:43:15,538 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,539 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:43:15,539 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,539 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:43:15,539 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,539 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,540 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,541 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:43:15,541 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c1644781f0b1f4c5e05c56d56502e2 columnFamilyName A 2024-11-28T07:43:15,541 DEBUG [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:15,542 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(327): Store=a4c1644781f0b1f4c5e05c56d56502e2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:43:15,542 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,543 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:43:15,543 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c1644781f0b1f4c5e05c56d56502e2 columnFamilyName B 2024-11-28T07:43:15,543 DEBUG [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:15,543 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(327): Store=a4c1644781f0b1f4c5e05c56d56502e2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:43:15,543 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,544 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:43:15,544 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c1644781f0b1f4c5e05c56d56502e2 columnFamilyName C 2024-11-28T07:43:15,544 DEBUG [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:15,545 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(327): Store=a4c1644781f0b1f4c5e05c56d56502e2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:43:15,545 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,545 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,545 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,546 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,546 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,546 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,547 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:43:15,548 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,549 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T07:43:15,550 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened a4c1644781f0b1f4c5e05c56d56502e2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65200650, jitterRate=-0.02843460440635681}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:43:15,550 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T07:43:15,550 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for a4c1644781f0b1f4c5e05c56d56502e2: Running coprocessor pre-open hook at 1732779795539Writing region info on filesystem at 1732779795539Initializing all the Stores at 1732779795540 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779795540Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779795540Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779795540Cleaning up temporary data from old regions at 1732779795546 (+6 ms)Running coprocessor post-open hooks at 1732779795550 (+4 ms)Region opened successfully at 1732779795550 2024-11-28T07:43:15,551 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., pid=146, masterSystemTime=1732779795536 2024-11-28T07:43:15,552 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,552 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:15,553 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=OPEN, openSeqNum=2, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:15,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:43:15,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-28T07:43:15,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 in 171 msec 2024-11-28T07:43:15,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-28T07:43:15,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, ASSIGN in 325 msec 2024-11-28T07:43:15,558 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T07:43:15,559 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779795558"}]},"ts":"1732779795558"} 2024-11-28T07:43:15,560 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-28T07:43:15,560 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T07:43:15,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-28T07:43:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-28T07:43:16,560 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:16,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:16,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:16,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:16,563 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c812665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:16,563 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:16,564 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:16,564 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6ba592, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:16,565 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:16,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:16,567 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T07:43:16,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f64cf8bdf19b,39063,1732779634677 2024-11-28T07:43:16,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@16f69f23 2024-11-28T07:43:16,568 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T07:43:16,568 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T07:43:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-28T07:43:16,570 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T07:43:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:16,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742365_1541 (size=1097) 2024-11-28T07:43:16,983 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-28T07:43:16,983 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-28T07:43:16,985 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:43:16,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, REOPEN/MOVE}] 2024-11-28T07:43:16,987 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, REOPEN/MOVE 2024-11-28T07:43:16,988 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:16,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, REOPEN/MOVE because future has completed 2024-11-28T07:43:16,989 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:43:16,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:43:17,141 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,141 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:43:17,141 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing a4c1644781f0b1f4c5e05c56d56502e2, disabling compactions & flushes 2024-11-28T07:43:17,141 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,141 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,141 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. after waiting 0 ms 2024-11-28T07:43:17,141 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,144 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-28T07:43:17,145 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,145 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for a4c1644781f0b1f4c5e05c56d56502e2: Waiting for close lock at 1732779797141Running coprocessor pre-close hooks at 1732779797141Disabling compacts and flushes for region at 1732779797141Disabling writes for close at 1732779797141Writing region close event to WAL at 1732779797142 (+1 ms)Running coprocessor post-close hooks at 1732779797145 (+3 ms)Closed at 1732779797145 2024-11-28T07:43:17,145 WARN [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: a4c1644781f0b1f4c5e05c56d56502e2 to self. 2024-11-28T07:43:17,146 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,147 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=CLOSED 2024-11-28T07:43:17,148 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:43:17,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-28T07:43:17,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 in 160 msec 2024-11-28T07:43:17,151 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, REOPEN/MOVE; state=CLOSED, location=f64cf8bdf19b,34437,1732779635619; forceNewPlan=false, retain=true 2024-11-28T07:43:17,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=OPENING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, REOPEN/MOVE because future has completed 2024-11-28T07:43:17,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:43:17,457 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,457 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} 2024-11-28T07:43:17,457 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,457 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T07:43:17,458 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,458 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,459 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,459 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:43:17,459 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c1644781f0b1f4c5e05c56d56502e2 columnFamilyName A 2024-11-28T07:43:17,460 DEBUG [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:17,461 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(327): Store=a4c1644781f0b1f4c5e05c56d56502e2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:43:17,461 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,461 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:43:17,462 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c1644781f0b1f4c5e05c56d56502e2 columnFamilyName B 2024-11-28T07:43:17,462 DEBUG [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:17,462 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(327): Store=a4c1644781f0b1f4c5e05c56d56502e2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:43:17,462 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,462 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-28T07:43:17,462 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c1644781f0b1f4c5e05c56d56502e2 columnFamilyName C 2024-11-28T07:43:17,463 DEBUG [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:17,463 INFO [StoreOpener-a4c1644781f0b1f4c5e05c56d56502e2-1 {}] regionserver.HStore(327): Store=a4c1644781f0b1f4c5e05c56d56502e2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T07:43:17,463 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,463 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,463 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,464 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,465 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,465 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,465 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-28T07:43:17,466 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,467 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened a4c1644781f0b1f4c5e05c56d56502e2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75383523, jitterRate=0.12330202758312225}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-28T07:43:17,467 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,467 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for a4c1644781f0b1f4c5e05c56d56502e2: Running coprocessor pre-open hook at 1732779797458Writing region info on filesystem at 1732779797458Initializing all the Stores at 1732779797458Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779797458Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779797458Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732779797458Cleaning up temporary data from old regions at 1732779797465 (+7 ms)Running coprocessor post-open hooks at 1732779797467 (+2 ms)Region opened successfully at 1732779797467 2024-11-28T07:43:17,468 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., pid=151, masterSystemTime=1732779797454 2024-11-28T07:43:17,469 DEBUG [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,470 INFO [RS_OPEN_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,470 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=OPEN, openSeqNum=5, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:43:17,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-28T07:43:17,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 in 169 msec 2024-11-28T07:43:17,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-28T07:43:17,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, REOPEN/MOVE in 486 msec 2024-11-28T07:43:17,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-28T07:43:17,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-28T07:43:17,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-28T07:43:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-28T07:43:17,479 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:17,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,480 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,481 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@891e18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,482 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c40cc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,483 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba47cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,485 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60886683, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,486 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2f4be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,487 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25287a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,488 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,489 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444338a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,490 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4327e314, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,491 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,491 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,491 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3adf69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,492 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522f37e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,494 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0fad84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2711adc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,496 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5df76c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,496 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,497 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,497 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720b9d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,498 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa65c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,499 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447559fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,500 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,501 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb90e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,502 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c751d97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a2803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f64cf8bdf19b,39063,-1 for getting cluster id 2024-11-28T07:43:17,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T07:43:17,503 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7ca35be7-5931-4a41-bd2b-792d46dc0434' 2024-11-28T07:43:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T07:43:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7ca35be7-5931-4a41-bd2b-792d46dc0434" 2024-11-28T07:43:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58bfc332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f64cf8bdf19b,39063,-1] 2024-11-28T07:43:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T07:43:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:17,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T07:43:17,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75afdce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T07:43:17,505 DEBUG [Thread-2432 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,505 DEBUG [Thread-2438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,505 DEBUG [Thread-2436 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,505 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,506 DEBUG [Thread-2421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,506 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:17,506 DEBUG [Thread-2423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-28T07:43:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T07:43:17,508 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,508 DEBUG [Thread-2425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,509 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:17,509 DEBUG [Thread-2434 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,510 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T07:43:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f64cf8bdf19b,34437,1732779635619, seqNum=-1] 2024-11-28T07:43:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T07:43:17,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,514 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T07:43:17,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5] 2024-11-28T07:43:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-28T07:43:17,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:17,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:17,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a034561eced94f2bac2f1d1fa856ce1a_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779797520/Put/seqid=0 2024-11-28T07:43:17,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779797529, max timestamp=1732779797544Num uniques 859; 2024-11-28T07:43:17,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732779797529, max timestamp=1732779797544Num uniques 859; 2024-11-28T07:43:17,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:43:17,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-28T07:43:17,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732779797540, max timestamp=1732779797544Num uniques 859; 2024-11-28T07:43:17,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-28T07:43:17,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742366_1542 (size=12154) 2024-11-28T07:43:17,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:17,559 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a034561eced94f2bac2f1d1fa856ce1a_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a034561eced94f2bac2f1d1fa856ce1a_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:17,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/2006bf362c8841138c6d98053a84828b, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:17,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/2006bf362c8841138c6d98053a84828b is 175, key is test_row_0/A:col10/1732779797520/Put/seqid=0 2024-11-28T07:43:17,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742367_1543 (size=30955) 2024-11-28T07:43:17,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779857560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779857560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779857561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779857565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779857565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T07:43:17,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-28T07:43:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:17,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779857670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779857670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779857670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779857680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779857680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-28T07:43:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T07:43:17,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779857880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779857880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779857880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779857890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779857890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:17,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:17,964 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/2006bf362c8841138c6d98053a84828b 2024-11-28T07:43:17,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-28T07:43:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:17,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:17,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/207810e9107d49dda8e3c16f9d293e3c is 50, key is test_row_0/B:col10/1732779797520/Put/seqid=0 2024-11-28T07:43:17,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742368_1544 (size=12001) 2024-11-28T07:43:17,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/207810e9107d49dda8e3c16f9d293e3c 2024-11-28T07:43:18,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/0a3be63851584edaba765494686463bf is 50, key is test_row_0/C:col10/1732779797520/Put/seqid=0 2024-11-28T07:43:18,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742369_1545 (size=12001) 2024-11-28T07:43:18,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/0a3be63851584edaba765494686463bf 2024-11-28T07:43:18,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/2006bf362c8841138c6d98053a84828b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/2006bf362c8841138c6d98053a84828b 2024-11-28T07:43:18,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/2006bf362c8841138c6d98053a84828b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-28T07:43:18,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/207810e9107d49dda8e3c16f9d293e3c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/207810e9107d49dda8e3c16f9d293e3c 2024-11-28T07:43:18,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/207810e9107d49dda8e3c16f9d293e3c, entries=150, sequenceid=16, filesize=11.7 K 2024-11-28T07:43:18,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/0a3be63851584edaba765494686463bf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/0a3be63851584edaba765494686463bf 2024-11-28T07:43:18,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/0a3be63851584edaba765494686463bf, entries=150, sequenceid=16, filesize=11.7 K 2024-11-28T07:43:18,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for a4c1644781f0b1f4c5e05c56d56502e2 in 512ms, sequenceid=16, compaction requested=false 2024-11-28T07:43:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-28T07:43:18,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:18,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-28T07:43:18,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:18,125 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=123.00 KB heapSize=345.53 KB 2024-11-28T07:43:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:18,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:18,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T07:43:18,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128893c77fa46e34f0899ada142f04beb2c_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779797563/Put/seqid=0 2024-11-28T07:43:18,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742370_1546 (size=12154) 2024-11-28T07:43:18,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:18,139 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128893c77fa46e34f0899ada142f04beb2c_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128893c77fa46e34f0899ada142f04beb2c_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:18,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/a407e18832664ea6bd88264030e6a2a6, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:18,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/a407e18832664ea6bd88264030e6a2a6 is 175, key is test_row_0/A:col10/1732779797563/Put/seqid=0 2024-11-28T07:43:18,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742371_1547 (size=30955) 2024-11-28T07:43:18,144 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/a407e18832664ea6bd88264030e6a2a6 2024-11-28T07:43:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/d999bf712b98469790e09340146ecff3 is 50, key is test_row_0/B:col10/1732779797563/Put/seqid=0 2024-11-28T07:43:18,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742372_1548 (size=12001) 2024-11-28T07:43:18,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:18,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:18,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779858202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779858207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779858208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779858208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779858208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779858320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779858320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779858321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779858321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779858321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779858530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779858530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779858531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779858531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779858531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,555 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/d999bf712b98469790e09340146ecff3 2024-11-28T07:43:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/83305daca0d44214b80ce78169adf3d1 is 50, key is test_row_0/C:col10/1732779797563/Put/seqid=0 2024-11-28T07:43:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742373_1549 (size=12001) 2024-11-28T07:43:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T07:43:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779858840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779858840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779858840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779858841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779858841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:18,966 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/83305daca0d44214b80ce78169adf3d1 2024-11-28T07:43:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/a407e18832664ea6bd88264030e6a2a6 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/a407e18832664ea6bd88264030e6a2a6 2024-11-28T07:43:18,973 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/a407e18832664ea6bd88264030e6a2a6, entries=150, sequenceid=50, filesize=30.2 K 2024-11-28T07:43:18,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/d999bf712b98469790e09340146ecff3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d999bf712b98469790e09340146ecff3 2024-11-28T07:43:18,977 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d999bf712b98469790e09340146ecff3, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T07:43:18,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/83305daca0d44214b80ce78169adf3d1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/83305daca0d44214b80ce78169adf3d1 2024-11-28T07:43:18,980 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/83305daca0d44214b80ce78169adf3d1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-28T07:43:18,981 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~345.48 KB/353776, currentSize=67.09 KB/68700 for a4c1644781f0b1f4c5e05c56d56502e2 in 857ms, sequenceid=50, compaction requested=false 2024-11-28T07:43:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-28T07:43:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-28T07:43:18,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-28T07:43:18,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-11-28T07:43:18,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4800 sec 2024-11-28T07:43:19,114 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:43:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:19,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:43:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:19,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:19,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112893f5bed8774147fa8d7e662f13e482d3_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779798206/Put/seqid=0 2024-11-28T07:43:19,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779859366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779859367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742374_1550 (size=12154) 2024-11-28T07:43:19,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779859368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779859370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779859371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779859480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779859480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779859480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779859480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779859480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-28T07:43:19,650 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:19,650 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-28T07:43:19,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T07:43:19,652 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:19,653 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:19,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:19,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779859690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779859690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779859690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779859691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779859691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:19,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T07:43:19,779 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:19,782 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112893f5bed8774147fa8d7e662f13e482d3_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112893f5bed8774147fa8d7e662f13e482d3_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:19,783 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/568ec2d30c8449d6a999548dd64edc95, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:19,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/568ec2d30c8449d6a999548dd64edc95 is 175, key is test_row_0/A:col10/1732779798206/Put/seqid=0 2024-11-28T07:43:19,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742375_1551 (size=30955) 2024-11-28T07:43:19,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:19,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:19,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:19,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:19,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T07:43:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779860000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779860000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779860001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779860001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779860001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,004 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/568ec2d30c8449d6a999548dd64edc95 2024-11-28T07:43:20,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/d749c6b2c2bd49fc9a089248dd64719d is 50, key is test_row_0/B:col10/1732779798206/Put/seqid=0 2024-11-28T07:43:20,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742376_1552 (size=12001) 2024-11-28T07:43:20,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T07:43:20,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779860510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779860510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779860510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779860510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779860510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:20,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/d749c6b2c2bd49fc9a089248dd64719d 2024-11-28T07:43:20,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/cacbc707620c48c895e3fdbb59112c03 is 50, key is test_row_0/C:col10/1732779798206/Put/seqid=0 2024-11-28T07:43:20,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742377_1553 (size=12001) 2024-11-28T07:43:20,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,722 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T07:43:20,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:20,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:20,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:20,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:20,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:21,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/cacbc707620c48c895e3fdbb59112c03 2024-11-28T07:43:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/568ec2d30c8449d6a999548dd64edc95 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/568ec2d30c8449d6a999548dd64edc95 2024-11-28T07:43:21,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/568ec2d30c8449d6a999548dd64edc95, entries=150, sequenceid=64, filesize=30.2 K 2024-11-28T07:43:21,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/d749c6b2c2bd49fc9a089248dd64719d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d749c6b2c2bd49fc9a089248dd64719d 2024-11-28T07:43:21,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d749c6b2c2bd49fc9a089248dd64719d, entries=150, sequenceid=64, filesize=11.7 K 2024-11-28T07:43:21,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/cacbc707620c48c895e3fdbb59112c03 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/cacbc707620c48c895e3fdbb59112c03 2024-11-28T07:43:21,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:21,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:21,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:21,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:21,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/cacbc707620c48c895e3fdbb59112c03, entries=150, sequenceid=64, filesize=11.7 K 2024-11-28T07:43:21,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:21,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a4c1644781f0b1f4c5e05c56d56502e2 in 1676ms, sequenceid=64, compaction requested=true 2024-11-28T07:43:21,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:21,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:21,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:21,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:21,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:21,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:21,029 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:21,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:21,029 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:21,030 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:21,030 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:21,030 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/B is initiating minor compaction (all files) 2024-11-28T07:43:21,030 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/A is initiating minor compaction (all files) 2024-11-28T07:43:21,030 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/A in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:21,030 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/B in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:21,030 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/2006bf362c8841138c6d98053a84828b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/a407e18832664ea6bd88264030e6a2a6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/568ec2d30c8449d6a999548dd64edc95] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=90.7 K 2024-11-28T07:43:21,030 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/207810e9107d49dda8e3c16f9d293e3c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d999bf712b98469790e09340146ecff3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d749c6b2c2bd49fc9a089248dd64719d] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.2 K 2024-11-28T07:43:21,030 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:21,030 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/2006bf362c8841138c6d98053a84828b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/a407e18832664ea6bd88264030e6a2a6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/568ec2d30c8449d6a999548dd64edc95] 2024-11-28T07:43:21,031 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 207810e9107d49dda8e3c16f9d293e3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732779797518 2024-11-28T07:43:21,031 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2006bf362c8841138c6d98053a84828b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732779797518 2024-11-28T07:43:21,031 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d999bf712b98469790e09340146ecff3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732779797557 2024-11-28T07:43:21,031 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting a407e18832664ea6bd88264030e6a2a6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732779797557 2024-11-28T07:43:21,031 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d749c6b2c2bd49fc9a089248dd64719d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732779798205 2024-11-28T07:43:21,032 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 568ec2d30c8449d6a999548dd64edc95, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732779798205 2024-11-28T07:43:21,042 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:21,043 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/3482875adfec4c2689126c4d947601bc is 50, key is test_row_0/B:col10/1732779798206/Put/seqid=0 2024-11-28T07:43:21,044 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:21,046 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112837f0a6705e6741be96f8e2de33a03d0f_a4c1644781f0b1f4c5e05c56d56502e2 store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:21,048 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112837f0a6705e6741be96f8e2de33a03d0f_a4c1644781f0b1f4c5e05c56d56502e2, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:21,048 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112837f0a6705e6741be96f8e2de33a03d0f_a4c1644781f0b1f4c5e05c56d56502e2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:21,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742378_1554 (size=12104) 2024-11-28T07:43:21,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742379_1555 (size=4469) 2024-11-28T07:43:21,060 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/3482875adfec4c2689126c4d947601bc as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/3482875adfec4c2689126c4d947601bc 2024-11-28T07:43:21,061 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#A#compaction#475 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:21,061 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/cb37bb49105946c699a00b983fc41bc4 is 175, key is test_row_0/A:col10/1732779798206/Put/seqid=0 2024-11-28T07:43:21,065 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/B of a4c1644781f0b1f4c5e05c56d56502e2 into 3482875adfec4c2689126c4d947601bc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:21,065 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:21,065 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/B, priority=13, startTime=1732779801029; duration=0sec 2024-11-28T07:43:21,065 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:21,065 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:B 2024-11-28T07:43:21,066 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:21,066 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:21,066 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/C is initiating minor compaction (all files) 2024-11-28T07:43:21,066 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/C in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:21,067 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/0a3be63851584edaba765494686463bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/83305daca0d44214b80ce78169adf3d1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/cacbc707620c48c895e3fdbb59112c03] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.2 K 2024-11-28T07:43:21,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742380_1556 (size=31058) 2024-11-28T07:43:21,067 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a3be63851584edaba765494686463bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732779797518 2024-11-28T07:43:21,067 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 83305daca0d44214b80ce78169adf3d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732779797557 2024-11-28T07:43:21,068 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting cacbc707620c48c895e3fdbb59112c03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732779798205 2024-11-28T07:43:21,072 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/cb37bb49105946c699a00b983fc41bc4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/cb37bb49105946c699a00b983fc41bc4 2024-11-28T07:43:21,076 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:21,077 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/3befffa778234cbaaa26bb5ba4976ad8 is 50, key is test_row_0/C:col10/1732779798206/Put/seqid=0 2024-11-28T07:43:21,078 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/A of a4c1644781f0b1f4c5e05c56d56502e2 into cb37bb49105946c699a00b983fc41bc4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:21,078 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:21,078 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/A, priority=13, startTime=1732779801029; duration=0sec 2024-11-28T07:43:21,078 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:21,078 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:A 2024-11-28T07:43:21,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742381_1557 (size=12104) 2024-11-28T07:43:21,084 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/3befffa778234cbaaa26bb5ba4976ad8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/3befffa778234cbaaa26bb5ba4976ad8 2024-11-28T07:43:21,087 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/C of a4c1644781f0b1f4c5e05c56d56502e2 into 3befffa778234cbaaa26bb5ba4976ad8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:21,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:21,087 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/C, priority=13, startTime=1732779801029; duration=0sec 2024-11-28T07:43:21,088 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:21,088 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:C 2024-11-28T07:43:21,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-28T07:43:21,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:21,180 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:43:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:21,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:21,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280f9b1d0b92d342a680a019f3a3fbcc77_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779799369/Put/seqid=0 2024-11-28T07:43:21,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742382_1558 (size=12154) 2024-11-28T07:43:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:21,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779861530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779861534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779861535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779861535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779861536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:21,594 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411280f9b1d0b92d342a680a019f3a3fbcc77_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280f9b1d0b92d342a680a019f3a3fbcc77_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d57a1486b3fc45ef99d266e9af138234, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d57a1486b3fc45ef99d266e9af138234 is 175, key is test_row_0/A:col10/1732779799369/Put/seqid=0 2024-11-28T07:43:21,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742383_1559 (size=30955) 2024-11-28T07:43:21,599 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d57a1486b3fc45ef99d266e9af138234 2024-11-28T07:43:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/6892bba265ce45eca67aa3e001bcb00e is 50, key is test_row_0/B:col10/1732779799369/Put/seqid=0 2024-11-28T07:43:21,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742384_1560 (size=12001) 2024-11-28T07:43:21,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779861640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779861649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779861650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779861650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779861650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T07:43:21,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779861850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779861860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779861860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779861860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779861860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:21,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,009 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/6892bba265ce45eca67aa3e001bcb00e 2024-11-28T07:43:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/30d73b6adca145e4a3f53e7d89e06e52 is 50, key is test_row_0/C:col10/1732779799369/Put/seqid=0 2024-11-28T07:43:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742385_1561 (size=12001) 2024-11-28T07:43:22,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779862160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779862170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779862170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779862170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779862170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,420 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/30d73b6adca145e4a3f53e7d89e06e52 2024-11-28T07:43:22,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d57a1486b3fc45ef99d266e9af138234 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d57a1486b3fc45ef99d266e9af138234 2024-11-28T07:43:22,426 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d57a1486b3fc45ef99d266e9af138234, entries=150, sequenceid=85, filesize=30.2 K 2024-11-28T07:43:22,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/6892bba265ce45eca67aa3e001bcb00e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/6892bba265ce45eca67aa3e001bcb00e 2024-11-28T07:43:22,430 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/6892bba265ce45eca67aa3e001bcb00e, entries=150, sequenceid=85, filesize=11.7 K 2024-11-28T07:43:22,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/30d73b6adca145e4a3f53e7d89e06e52 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/30d73b6adca145e4a3f53e7d89e06e52 2024-11-28T07:43:22,434 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/30d73b6adca145e4a3f53e7d89e06e52, entries=150, sequenceid=85, filesize=11.7 K 2024-11-28T07:43:22,435 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for a4c1644781f0b1f4c5e05c56d56502e2 in 1255ms, sequenceid=85, compaction requested=false 2024-11-28T07:43:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:22,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-28T07:43:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-28T07:43:22,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-28T07:43:22,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7830 sec 2024-11-28T07:43:22,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7880 sec 2024-11-28T07:43:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:22,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:43:22,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:22,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:22,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:22,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:22,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:22,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:22,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284d5e18c11cc3414496e102349742471b_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779802671/Put/seqid=0 2024-11-28T07:43:22,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742386_1562 (size=14594) 2024-11-28T07:43:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779862689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779862690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779862691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779862693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779862693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779862800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779862810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779862810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779862810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779862810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:22,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779863010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779863020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779863020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779863020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779863021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,084 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:23,087 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411284d5e18c11cc3414496e102349742471b_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284d5e18c11cc3414496e102349742471b_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:23,088 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e013d474b7944697a59054c60915ec71, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:23,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e013d474b7944697a59054c60915ec71 is 175, key is test_row_0/A:col10/1732779802671/Put/seqid=0 2024-11-28T07:43:23,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742387_1563 (size=39549) 2024-11-28T07:43:23,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779863320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779863330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779863330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779863330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779863330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,492 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e013d474b7944697a59054c60915ec71 2024-11-28T07:43:23,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/c124b43a1baa44149156875eb80a169c is 50, key is test_row_0/B:col10/1732779802671/Put/seqid=0 2024-11-28T07:43:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742388_1564 (size=12001) 2024-11-28T07:43:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-28T07:43:23,810 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:23,811 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-28T07:43:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T07:43:23,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:23,813 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:23,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:23,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779863830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779863840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779863840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779863841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779863841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:23,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/c124b43a1baa44149156875eb80a169c 2024-11-28T07:43:23,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/8c4c1cb0d9e042f1b8ac85ffb7005856 is 50, key is test_row_0/C:col10/1732779802671/Put/seqid=0 2024-11-28T07:43:23,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742389_1565 (size=12001) 2024-11-28T07:43:23,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T07:43:23,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T07:43:23,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:23,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:24,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T07:43:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:24,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:24,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T07:43:24,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T07:43:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:24,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:24,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:24,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/8c4c1cb0d9e042f1b8ac85ffb7005856 2024-11-28T07:43:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e013d474b7944697a59054c60915ec71 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e013d474b7944697a59054c60915ec71 2024-11-28T07:43:24,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e013d474b7944697a59054c60915ec71, entries=200, sequenceid=101, filesize=38.6 K 2024-11-28T07:43:24,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/c124b43a1baa44149156875eb80a169c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c124b43a1baa44149156875eb80a169c 2024-11-28T07:43:24,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c124b43a1baa44149156875eb80a169c, entries=150, sequenceid=101, filesize=11.7 K 2024-11-28T07:43:24,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/8c4c1cb0d9e042f1b8ac85ffb7005856 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/8c4c1cb0d9e042f1b8ac85ffb7005856 2024-11-28T07:43:24,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/8c4c1cb0d9e042f1b8ac85ffb7005856, entries=150, sequenceid=101, filesize=11.7 K 2024-11-28T07:43:24,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a4c1644781f0b1f4c5e05c56d56502e2 in 1657ms, sequenceid=101, compaction requested=true 2024-11-28T07:43:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:24,329 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:24,329 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:24,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:24,330 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:24,330 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/B is initiating minor compaction (all files) 2024-11-28T07:43:24,330 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:24,330 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/B in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,330 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/A is initiating minor compaction (all files) 2024-11-28T07:43:24,330 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/A in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,330 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/3482875adfec4c2689126c4d947601bc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/6892bba265ce45eca67aa3e001bcb00e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c124b43a1baa44149156875eb80a169c] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.3 K 2024-11-28T07:43:24,330 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/cb37bb49105946c699a00b983fc41bc4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d57a1486b3fc45ef99d266e9af138234, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e013d474b7944697a59054c60915ec71] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=99.2 K 2024-11-28T07:43:24,330 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,330 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/cb37bb49105946c699a00b983fc41bc4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d57a1486b3fc45ef99d266e9af138234, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e013d474b7944697a59054c60915ec71] 2024-11-28T07:43:24,331 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb37bb49105946c699a00b983fc41bc4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732779798205 2024-11-28T07:43:24,331 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 3482875adfec4c2689126c4d947601bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732779798205 2024-11-28T07:43:24,331 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting d57a1486b3fc45ef99d266e9af138234, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732779799364 2024-11-28T07:43:24,331 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 6892bba265ce45eca67aa3e001bcb00e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732779799364 2024-11-28T07:43:24,331 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting c124b43a1baa44149156875eb80a169c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732779801533 2024-11-28T07:43:24,331 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e013d474b7944697a59054c60915ec71, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732779801533 2024-11-28T07:43:24,336 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:24,338 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:24,338 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/91ab40d28adb45a7bfc8387cb7d6c0db is 50, key is test_row_0/B:col10/1732779802671/Put/seqid=0 2024-11-28T07:43:24,345 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128b9d3e830216d458893b2ed9a9bbeaf0d_a4c1644781f0b1f4c5e05c56d56502e2 store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:24,346 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128b9d3e830216d458893b2ed9a9bbeaf0d_a4c1644781f0b1f4c5e05c56d56502e2, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:24,346 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128b9d3e830216d458893b2ed9a9bbeaf0d_a4c1644781f0b1f4c5e05c56d56502e2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:24,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742390_1566 (size=12207) 2024-11-28T07:43:24,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742391_1567 (size=4469) 2024-11-28T07:43:24,361 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#A#compaction#483 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:24,361 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c203b35e12e94d1591bca41a449ec1c4 is 175, key is test_row_0/A:col10/1732779802671/Put/seqid=0 2024-11-28T07:43:24,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742392_1568 (size=31161) 2024-11-28T07:43:24,370 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c203b35e12e94d1591bca41a449ec1c4 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c203b35e12e94d1591bca41a449ec1c4 2024-11-28T07:43:24,375 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/A of a4c1644781f0b1f4c5e05c56d56502e2 into c203b35e12e94d1591bca41a449ec1c4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:24,375 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:24,375 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/A, priority=13, startTime=1732779804329; duration=0sec 2024-11-28T07:43:24,375 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:24,375 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:A 2024-11-28T07:43:24,375 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:24,376 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:24,376 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/C is initiating minor compaction (all files) 2024-11-28T07:43:24,376 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/C in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,376 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/3befffa778234cbaaa26bb5ba4976ad8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/30d73b6adca145e4a3f53e7d89e06e52, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/8c4c1cb0d9e042f1b8ac85ffb7005856] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.3 K 2024-11-28T07:43:24,377 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3befffa778234cbaaa26bb5ba4976ad8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732779798205 2024-11-28T07:43:24,377 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30d73b6adca145e4a3f53e7d89e06e52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732779799364 2024-11-28T07:43:24,377 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c4c1cb0d9e042f1b8ac85ffb7005856, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732779801533 2024-11-28T07:43:24,385 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#C#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:24,385 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/2fc784aaa64943ebbd69eb66a7411fdd is 50, key is test_row_0/C:col10/1732779802671/Put/seqid=0 2024-11-28T07:43:24,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742393_1569 (size=12207) 2024-11-28T07:43:24,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-28T07:43:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:24,424 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-28T07:43:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:24,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282cfd645bc3904701a4692b900042aedc_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779802690/Put/seqid=0 2024-11-28T07:43:24,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742394_1570 (size=12154) 2024-11-28T07:43:24,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:24,437 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282cfd645bc3904701a4692b900042aedc_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282cfd645bc3904701a4692b900042aedc_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:24,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e91f045ccd7c4ebfbda3dd536d5b0439, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:24,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e91f045ccd7c4ebfbda3dd536d5b0439 is 175, key is test_row_0/A:col10/1732779802690/Put/seqid=0 2024-11-28T07:43:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T07:43:24,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742395_1571 (size=30955) 2024-11-28T07:43:24,754 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/91ab40d28adb45a7bfc8387cb7d6c0db as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/91ab40d28adb45a7bfc8387cb7d6c0db 2024-11-28T07:43:24,758 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/B of a4c1644781f0b1f4c5e05c56d56502e2 into 91ab40d28adb45a7bfc8387cb7d6c0db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:24,758 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:24,758 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/B, priority=13, startTime=1732779804329; duration=0sec 2024-11-28T07:43:24,758 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:24,758 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:B 2024-11-28T07:43:24,792 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/2fc784aaa64943ebbd69eb66a7411fdd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2fc784aaa64943ebbd69eb66a7411fdd 2024-11-28T07:43:24,796 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/C of a4c1644781f0b1f4c5e05c56d56502e2 into 2fc784aaa64943ebbd69eb66a7411fdd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:24,796 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:24,796 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/C, priority=13, startTime=1732779804329; duration=0sec 2024-11-28T07:43:24,796 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:24,796 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:C 2024-11-28T07:43:24,842 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e91f045ccd7c4ebfbda3dd536d5b0439 2024-11-28T07:43:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:24,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf is 50, key is test_row_0/B:col10/1732779802690/Put/seqid=0 2024-11-28T07:43:24,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742396_1572 (size=12001) 2024-11-28T07:43:24,854 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf 2024-11-28T07:43:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/4ac8a0294e384e66ba83b209ed3a5292 is 50, key is test_row_0/C:col10/1732779802690/Put/seqid=0 2024-11-28T07:43:24,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779864862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779864863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779864863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779864866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742397_1573 (size=12001) 2024-11-28T07:43:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779864867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,871 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T07:43:24,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779864970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779864970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779864980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779864980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779864980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T07:43:25,113 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-28T07:43:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779865180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779865180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779865190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779865190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779865190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,271 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/4ac8a0294e384e66ba83b209ed3a5292 2024-11-28T07:43:25,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e91f045ccd7c4ebfbda3dd536d5b0439 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e91f045ccd7c4ebfbda3dd536d5b0439 2024-11-28T07:43:25,278 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e91f045ccd7c4ebfbda3dd536d5b0439, entries=150, sequenceid=118, filesize=30.2 K 2024-11-28T07:43:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf 2024-11-28T07:43:25,282 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf, entries=150, sequenceid=118, filesize=11.7 K 2024-11-28T07:43:25,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/4ac8a0294e384e66ba83b209ed3a5292 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/4ac8a0294e384e66ba83b209ed3a5292 2024-11-28T07:43:25,286 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/4ac8a0294e384e66ba83b209ed3a5292, entries=150, sequenceid=118, filesize=11.7 K 2024-11-28T07:43:25,286 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a4c1644781f0b1f4c5e05c56d56502e2 in 863ms, sequenceid=118, compaction requested=false 2024-11-28T07:43:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-28T07:43:25,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-28T07:43:25,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-28T07:43:25,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4760 sec 2024-11-28T07:43:25,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4800 sec 2024-11-28T07:43:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:25,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:43:25,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:25,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:25,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:25,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:25,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:25,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:25,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d4575e996bbf4ece9e87ac6403e041ba_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779805491/Put/seqid=0 2024-11-28T07:43:25,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742398_1574 (size=17284) 2024-11-28T07:43:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779865506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779865507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779865508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779865511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779865512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779865620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779865620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779865621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779865621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779865621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,624 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779865830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779865830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779865830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779865830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779865830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:25,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:25,904 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:25,907 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128d4575e996bbf4ece9e87ac6403e041ba_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4575e996bbf4ece9e87ac6403e041ba_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:25,908 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/16b25389764e48d6bea098cce6deff66, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:25,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/16b25389764e48d6bea098cce6deff66 is 175, key is test_row_0/A:col10/1732779805491/Put/seqid=0 2024-11-28T07:43:25,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742399_1575 (size=48389) 2024-11-28T07:43:25,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/16b25389764e48d6bea098cce6deff66 2024-11-28T07:43:25,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/f701f182a2604ec79f8de90ad07763aa is 50, key is test_row_0/B:col10/1732779805491/Put/seqid=0 2024-11-28T07:43:25,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742400_1576 (size=12151) 2024-11-28T07:43:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-28T07:43:25,961 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:25,961 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-28T07:43:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T07:43:25,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:25,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:25,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T07:43:26,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T07:43:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:26,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779866141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779866141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779866142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779866143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779866143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T07:43:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T07:43:26,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/f701f182a2604ec79f8de90ad07763aa 2024-11-28T07:43:26,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/25213a27c60a48d3a9f1d1ae98035e21 is 50, key is test_row_0/C:col10/1732779805491/Put/seqid=0 2024-11-28T07:43:26,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742401_1577 (size=12151) 2024-11-28T07:43:26,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T07:43:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T07:43:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T07:43:26,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779866650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779866650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779866661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779866661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779866661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:26,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:26,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T07:43:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:26,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:26,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/25213a27c60a48d3a9f1d1ae98035e21 2024-11-28T07:43:26,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/16b25389764e48d6bea098cce6deff66 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/16b25389764e48d6bea098cce6deff66 2024-11-28T07:43:26,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/16b25389764e48d6bea098cce6deff66, entries=250, sequenceid=138, filesize=47.3 K 2024-11-28T07:43:26,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/f701f182a2604ec79f8de90ad07763aa as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/f701f182a2604ec79f8de90ad07763aa 2024-11-28T07:43:26,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/f701f182a2604ec79f8de90ad07763aa, entries=150, sequenceid=138, filesize=11.9 K 2024-11-28T07:43:26,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/25213a27c60a48d3a9f1d1ae98035e21 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/25213a27c60a48d3a9f1d1ae98035e21 2024-11-28T07:43:26,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/25213a27c60a48d3a9f1d1ae98035e21, entries=150, sequenceid=138, filesize=11.9 K 2024-11-28T07:43:26,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for a4c1644781f0b1f4c5e05c56d56502e2 in 1271ms, sequenceid=138, compaction requested=true 2024-11-28T07:43:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:26,763 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:26,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:26,763 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:26,764 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:26,764 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/A is initiating minor compaction (all files) 2024-11-28T07:43:26,764 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/A in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,764 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c203b35e12e94d1591bca41a449ec1c4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e91f045ccd7c4ebfbda3dd536d5b0439, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/16b25389764e48d6bea098cce6deff66] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=107.9 K 2024-11-28T07:43:26,765 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,765 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c203b35e12e94d1591bca41a449ec1c4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e91f045ccd7c4ebfbda3dd536d5b0439, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/16b25389764e48d6bea098cce6deff66] 2024-11-28T07:43:26,765 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:26,765 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/B is initiating minor compaction (all files) 2024-11-28T07:43:26,765 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/B in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,765 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/91ab40d28adb45a7bfc8387cb7d6c0db, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/f701f182a2604ec79f8de90ad07763aa] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.5 K 2024-11-28T07:43:26,765 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c203b35e12e94d1591bca41a449ec1c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732779801533 2024-11-28T07:43:26,766 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 91ab40d28adb45a7bfc8387cb7d6c0db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732779801533 2024-11-28T07:43:26,766 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91f045ccd7c4ebfbda3dd536d5b0439, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732779802690 2024-11-28T07:43:26,766 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc7642dd8bc4ce1b1e5f534ea7a15cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732779802690 2024-11-28T07:43:26,766 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b25389764e48d6bea098cce6deff66, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732779804860 2024-11-28T07:43:26,767 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting f701f182a2604ec79f8de90ad07763aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732779804860 2024-11-28T07:43:26,778 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#B#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:26,778 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/bf99ccb332084f49bb3dea8e2be1f64a is 50, key is test_row_0/B:col10/1732779805491/Put/seqid=0 2024-11-28T07:43:26,781 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:26,784 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128c2fb558af50f4f51968a5c49032d1248_a4c1644781f0b1f4c5e05c56d56502e2 store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:26,787 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128c2fb558af50f4f51968a5c49032d1248_a4c1644781f0b1f4c5e05c56d56502e2, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:26,787 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128c2fb558af50f4f51968a5c49032d1248_a4c1644781f0b1f4c5e05c56d56502e2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:26,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742402_1578 (size=12459) 2024-11-28T07:43:26,814 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/bf99ccb332084f49bb3dea8e2be1f64a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/bf99ccb332084f49bb3dea8e2be1f64a 2024-11-28T07:43:26,820 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/B of a4c1644781f0b1f4c5e05c56d56502e2 into bf99ccb332084f49bb3dea8e2be1f64a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:26,820 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:26,820 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/B, priority=13, startTime=1732779806763; duration=0sec 2024-11-28T07:43:26,821 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:26,821 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:B 2024-11-28T07:43:26,821 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:26,823 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:26,823 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/C is initiating minor compaction (all files) 2024-11-28T07:43:26,823 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/C in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,823 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2fc784aaa64943ebbd69eb66a7411fdd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/4ac8a0294e384e66ba83b209ed3a5292, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/25213a27c60a48d3a9f1d1ae98035e21] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.5 K 2024-11-28T07:43:26,824 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fc784aaa64943ebbd69eb66a7411fdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732779801533 2024-11-28T07:43:26,824 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac8a0294e384e66ba83b209ed3a5292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732779802690 2024-11-28T07:43:26,824 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 25213a27c60a48d3a9f1d1ae98035e21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732779804860 2024-11-28T07:43:26,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742403_1579 (size=4469) 2024-11-28T07:43:26,832 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#A#compaction#493 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:26,832 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/8ec91562be6949fe91d17b65134a2928 is 175, key is test_row_0/A:col10/1732779805491/Put/seqid=0 2024-11-28T07:43:26,846 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:26,846 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/9c3a26dc13cd4bff9c908cf24fb8b289 is 50, key is test_row_0/C:col10/1732779805491/Put/seqid=0 2024-11-28T07:43:26,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742404_1580 (size=31413) 2024-11-28T07:43:26,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742405_1581 (size=12459) 2024-11-28T07:43:26,879 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/8ec91562be6949fe91d17b65134a2928 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/8ec91562be6949fe91d17b65134a2928 2024-11-28T07:43:26,881 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/9c3a26dc13cd4bff9c908cf24fb8b289 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/9c3a26dc13cd4bff9c908cf24fb8b289 2024-11-28T07:43:26,885 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/A of a4c1644781f0b1f4c5e05c56d56502e2 into 8ec91562be6949fe91d17b65134a2928(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:26,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-28T07:43:26,885 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:26,885 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/A, priority=13, startTime=1732779806763; duration=0sec 2024-11-28T07:43:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:26,886 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:26,886 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:A 2024-11-28T07:43:26,886 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-28T07:43:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:26,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:26,887 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/C of a4c1644781f0b1f4c5e05c56d56502e2 into 9c3a26dc13cd4bff9c908cf24fb8b289(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:26,887 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:26,887 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/C, priority=13, startTime=1732779806763; duration=0sec 2024-11-28T07:43:26,887 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:26,887 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:C 2024-11-28T07:43:26,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128deff424f8337435bbb1803628782a79b_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779805506/Put/seqid=0 2024-11-28T07:43:26,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742406_1582 (size=12304) 2024-11-28T07:43:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T07:43:27,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:27,316 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128deff424f8337435bbb1803628782a79b_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128deff424f8337435bbb1803628782a79b_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e4e87b5cd9c94f9db319d94009d114f8, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:27,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e4e87b5cd9c94f9db319d94009d114f8 is 175, key is test_row_0/A:col10/1732779805506/Put/seqid=0 2024-11-28T07:43:27,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742407_1583 (size=31105) 2024-11-28T07:43:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:27,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779867685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779867686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779867687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779867689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779867690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,721 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e4e87b5cd9c94f9db319d94009d114f8 2024-11-28T07:43:27,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/2a52e709f8c94c04b4031b68d566c343 is 50, key is test_row_0/B:col10/1732779805506/Put/seqid=0 2024-11-28T07:43:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742408_1584 (size=12151) 2024-11-28T07:43:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779867800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779867800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779867801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779867801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779867801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779868011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779868011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779868011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779868011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779868011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T07:43:28,134 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/2a52e709f8c94c04b4031b68d566c343 2024-11-28T07:43:28,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/69fee21b2bb243649d004b1b6ca44164 is 50, key is test_row_0/C:col10/1732779805506/Put/seqid=0 2024-11-28T07:43:28,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742409_1585 (size=12151) 2024-11-28T07:43:28,144 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/69fee21b2bb243649d004b1b6ca44164 2024-11-28T07:43:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/e4e87b5cd9c94f9db319d94009d114f8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e4e87b5cd9c94f9db319d94009d114f8 2024-11-28T07:43:28,150 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e4e87b5cd9c94f9db319d94009d114f8, entries=150, sequenceid=155, filesize=30.4 K 2024-11-28T07:43:28,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/2a52e709f8c94c04b4031b68d566c343 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/2a52e709f8c94c04b4031b68d566c343 2024-11-28T07:43:28,154 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/2a52e709f8c94c04b4031b68d566c343, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T07:43:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/69fee21b2bb243649d004b1b6ca44164 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/69fee21b2bb243649d004b1b6ca44164 2024-11-28T07:43:28,162 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/69fee21b2bb243649d004b1b6ca44164, entries=150, sequenceid=155, filesize=11.9 K 2024-11-28T07:43:28,163 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a4c1644781f0b1f4c5e05c56d56502e2 in 1277ms, sequenceid=155, compaction requested=false 2024-11-28T07:43:28,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:28,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:28,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-28T07:43:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-28T07:43:28,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-28T07:43:28,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2000 sec 2024-11-28T07:43:28,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.2050 sec 2024-11-28T07:43:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:28,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-28T07:43:28,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:28,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:28,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:28,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:28,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:28,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:28,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128587a2cd5cdc8456ba77a8d70b15c0f63_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779807685/Put/seqid=0 2024-11-28T07:43:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742410_1586 (size=12304) 2024-11-28T07:43:28,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779868332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779868333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779868333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779868338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779868338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779868450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779868450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779868450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779868450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779868450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779868660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779868660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779868660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779868661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779868661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:28,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128587a2cd5cdc8456ba77a8d70b15c0f63_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128587a2cd5cdc8456ba77a8d70b15c0f63_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:28,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/195386d9c9ba42798ace9f955c22c3da, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:28,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/195386d9c9ba42798ace9f955c22c3da is 175, key is test_row_0/A:col10/1732779807685/Put/seqid=0 2024-11-28T07:43:28,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742411_1587 (size=31105) 2024-11-28T07:43:28,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779868970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779868971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779868971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779868971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779868971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:29,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/195386d9c9ba42798ace9f955c22c3da 2024-11-28T07:43:29,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/71aa1299dbcb4be48e8e10bb19511375 is 50, key is test_row_0/B:col10/1732779807685/Put/seqid=0 2024-11-28T07:43:29,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742412_1588 (size=12151) 2024-11-28T07:43:29,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/71aa1299dbcb4be48e8e10bb19511375 2024-11-28T07:43:29,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/bd3037de70c6472387083625c18cb2dd is 50, key is test_row_0/C:col10/1732779807685/Put/seqid=0 2024-11-28T07:43:29,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742413_1589 (size=12151) 2024-11-28T07:43:29,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779869480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779869480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:29,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779869480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779869481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779869481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:29,485 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:29,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/bd3037de70c6472387083625c18cb2dd 2024-11-28T07:43:29,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/195386d9c9ba42798ace9f955c22c3da as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/195386d9c9ba42798ace9f955c22c3da 2024-11-28T07:43:29,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/195386d9c9ba42798ace9f955c22c3da, entries=150, sequenceid=176, filesize=30.4 K 2024-11-28T07:43:29,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/71aa1299dbcb4be48e8e10bb19511375 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/71aa1299dbcb4be48e8e10bb19511375 2024-11-28T07:43:29,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/71aa1299dbcb4be48e8e10bb19511375, entries=150, sequenceid=176, filesize=11.9 K 2024-11-28T07:43:29,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/bd3037de70c6472387083625c18cb2dd as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/bd3037de70c6472387083625c18cb2dd 2024-11-28T07:43:29,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/bd3037de70c6472387083625c18cb2dd, entries=150, sequenceid=176, filesize=11.9 K 2024-11-28T07:43:29,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a4c1644781f0b1f4c5e05c56d56502e2 in 1263ms, sequenceid=176, compaction requested=true 2024-11-28T07:43:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:29,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:29,587 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:29,587 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:29,588 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:29,588 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/A is initiating minor compaction (all files) 2024-11-28T07:43:29,588 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/A in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:29,588 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/8ec91562be6949fe91d17b65134a2928, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e4e87b5cd9c94f9db319d94009d114f8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/195386d9c9ba42798ace9f955c22c3da] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=91.4 K 2024-11-28T07:43:29,588 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:29,588 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/8ec91562be6949fe91d17b65134a2928, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e4e87b5cd9c94f9db319d94009d114f8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/195386d9c9ba42798ace9f955c22c3da] 2024-11-28T07:43:29,589 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:29,589 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/B is initiating minor compaction (all files) 2024-11-28T07:43:29,589 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/B in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:29,589 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/bf99ccb332084f49bb3dea8e2be1f64a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/2a52e709f8c94c04b4031b68d566c343, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/71aa1299dbcb4be48e8e10bb19511375] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.9 K 2024-11-28T07:43:29,589 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ec91562be6949fe91d17b65134a2928, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732779804860 2024-11-28T07:43:29,589 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting bf99ccb332084f49bb3dea8e2be1f64a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732779804860 2024-11-28T07:43:29,589 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e87b5cd9c94f9db319d94009d114f8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779805505 2024-11-28T07:43:29,589 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 195386d9c9ba42798ace9f955c22c3da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779807685 2024-11-28T07:43:29,590 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a52e709f8c94c04b4031b68d566c343, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779805505 2024-11-28T07:43:29,590 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 71aa1299dbcb4be48e8e10bb19511375, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779807685 2024-11-28T07:43:29,594 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:29,596 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#B#compaction#502 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:29,596 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/c56e7e91b7aa4f65a2cf4829447b9d07 is 50, key is test_row_0/B:col10/1732779807685/Put/seqid=0 2024-11-28T07:43:29,596 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411281a2f374b5ceb46b995f4c24258fadebf_a4c1644781f0b1f4c5e05c56d56502e2 store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:29,598 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411281a2f374b5ceb46b995f4c24258fadebf_a4c1644781f0b1f4c5e05c56d56502e2, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:29,598 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411281a2f374b5ceb46b995f4c24258fadebf_a4c1644781f0b1f4c5e05c56d56502e2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:29,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742414_1590 (size=12561) 2024-11-28T07:43:29,603 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/c56e7e91b7aa4f65a2cf4829447b9d07 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c56e7e91b7aa4f65a2cf4829447b9d07 2024-11-28T07:43:29,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742415_1591 (size=4469) 2024-11-28T07:43:29,607 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/B of a4c1644781f0b1f4c5e05c56d56502e2 into c56e7e91b7aa4f65a2cf4829447b9d07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:29,607 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:29,607 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/B, priority=13, startTime=1732779809587; duration=0sec 2024-11-28T07:43:29,607 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:29,607 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:B 2024-11-28T07:43:29,607 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:29,608 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:29,608 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/C is initiating minor compaction (all files) 2024-11-28T07:43:29,608 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/C in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:29,608 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/9c3a26dc13cd4bff9c908cf24fb8b289, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/69fee21b2bb243649d004b1b6ca44164, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/bd3037de70c6472387083625c18cb2dd] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=35.9 K 2024-11-28T07:43:29,609 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c3a26dc13cd4bff9c908cf24fb8b289, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1732779804860 2024-11-28T07:43:29,609 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 69fee21b2bb243649d004b1b6ca44164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732779805505 2024-11-28T07:43:29,609 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3037de70c6472387083625c18cb2dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779807685 2024-11-28T07:43:29,616 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:29,616 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/1fba934ad1fb4a88a8d2e25493f2bad1 is 50, key is test_row_0/C:col10/1732779807685/Put/seqid=0 2024-11-28T07:43:29,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742416_1592 (size=12561) 2024-11-28T07:43:30,008 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:30,008 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/140c71e4ad0249bfadf581de3b13e03b is 175, key is test_row_0/A:col10/1732779807685/Put/seqid=0 2024-11-28T07:43:30,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742417_1593 (size=31515) 2024-11-28T07:43:30,016 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/140c71e4ad0249bfadf581de3b13e03b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/140c71e4ad0249bfadf581de3b13e03b 2024-11-28T07:43:30,020 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/A of a4c1644781f0b1f4c5e05c56d56502e2 into 140c71e4ad0249bfadf581de3b13e03b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:30,020 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:30,021 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/A, priority=13, startTime=1732779809587; duration=0sec 2024-11-28T07:43:30,021 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:30,021 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:A 2024-11-28T07:43:30,026 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/1fba934ad1fb4a88a8d2e25493f2bad1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1fba934ad1fb4a88a8d2e25493f2bad1 2024-11-28T07:43:30,031 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/C of a4c1644781f0b1f4c5e05c56d56502e2 into 1fba934ad1fb4a88a8d2e25493f2bad1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:30,031 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:30,031 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/C, priority=13, startTime=1732779809587; duration=0sec 2024-11-28T07:43:30,031 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:30,031 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:C 2024-11-28T07:43:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-28T07:43:30,121 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:30,121 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-28T07:43:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T07:43:30,123 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:30,124 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:30,124 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T07:43:30,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-28T07:43:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:30,277 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:43:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:30,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:30,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289c09cb722102496890e9d2b64a7b416f_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779808331/Put/seqid=0 2024-11-28T07:43:30,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742418_1594 (size=12304) 2024-11-28T07:43:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T07:43:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:30,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:30,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779870540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779870544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779870545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779870545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779870547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779870650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779870660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779870660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779870660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779870661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:30,691 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411289c09cb722102496890e9d2b64a7b416f_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289c09cb722102496890e9d2b64a7b416f_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/826cc035aa974f969600bcab56d9d4bf, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:30,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/826cc035aa974f969600bcab56d9d4bf is 175, key is test_row_0/A:col10/1732779808331/Put/seqid=0 2024-11-28T07:43:30,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742419_1595 (size=31105) 2024-11-28T07:43:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T07:43:30,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779870860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779870870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779870870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779870871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:30,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:30,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779870871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,096 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/826cc035aa974f969600bcab56d9d4bf 2024-11-28T07:43:31,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/945009cc4f6e463686577968e67d866d is 50, key is test_row_0/B:col10/1732779808331/Put/seqid=0 2024-11-28T07:43:31,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742420_1596 (size=12151) 2024-11-28T07:43:31,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779871170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779871180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779871180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779871190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779871191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T07:43:31,509 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/945009cc4f6e463686577968e67d866d 2024-11-28T07:43:31,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/633d77acd63946808a8ee5d3fe51135e is 50, key is test_row_0/C:col10/1732779808331/Put/seqid=0 2024-11-28T07:43:31,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742421_1597 (size=12151) 2024-11-28T07:43:31,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779871680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779871690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779871690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779871700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779871700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:31,934 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/633d77acd63946808a8ee5d3fe51135e 2024-11-28T07:43:31,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/826cc035aa974f969600bcab56d9d4bf as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/826cc035aa974f969600bcab56d9d4bf 2024-11-28T07:43:31,941 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/826cc035aa974f969600bcab56d9d4bf, entries=150, sequenceid=191, filesize=30.4 K 2024-11-28T07:43:31,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/945009cc4f6e463686577968e67d866d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/945009cc4f6e463686577968e67d866d 2024-11-28T07:43:31,945 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/945009cc4f6e463686577968e67d866d, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T07:43:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/633d77acd63946808a8ee5d3fe51135e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/633d77acd63946808a8ee5d3fe51135e 2024-11-28T07:43:31,949 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/633d77acd63946808a8ee5d3fe51135e, entries=150, sequenceid=191, filesize=11.9 K 2024-11-28T07:43:31,950 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a4c1644781f0b1f4c5e05c56d56502e2 in 1673ms, sequenceid=191, compaction requested=false 2024-11-28T07:43:31,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:31,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:31,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-28T07:43:31,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-28T07:43:31,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-28T07:43:31,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8270 sec 2024-11-28T07:43:31,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8320 sec 2024-11-28T07:43:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-28T07:43:32,270 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:32,270 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-28T07:43:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T07:43:32,273 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:32,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:32,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T07:43:32,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-28T07:43:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:32,427 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-28T07:43:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:32,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288e9ed67d16bb49bfab7963a99c24d4f6_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779810544/Put/seqid=0 2024-11-28T07:43:32,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742422_1598 (size=12304) 2024-11-28T07:43:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T07:43:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:32,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:32,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779872737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779872737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779872737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779872738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779872738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,743 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:32,842 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411288e9ed67d16bb49bfab7963a99c24d4f6_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288e9ed67d16bb49bfab7963a99c24d4f6_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:32,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ffc3abe0ee4c4cd185ad96c459ccfa94, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:32,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ffc3abe0ee4c4cd185ad96c459ccfa94 is 175, key is test_row_0/A:col10/1732779810544/Put/seqid=0 2024-11-28T07:43:32,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742423_1599 (size=31105) 2024-11-28T07:43:32,847 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ffc3abe0ee4c4cd185ad96c459ccfa94 2024-11-28T07:43:32,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779872850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779872850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779872850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/4f17a36da6584758bd32ee53596e08ac is 50, key is test_row_0/B:col10/1732779810544/Put/seqid=0 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779872850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779872850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:32,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742424_1600 (size=12151) 2024-11-28T07:43:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T07:43:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779873060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779873060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779873060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779873060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779873060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/4f17a36da6584758bd32ee53596e08ac 2024-11-28T07:43:33,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/967744d390ec46609f0bf35c200ef129 is 50, key is test_row_0/C:col10/1732779810544/Put/seqid=0 2024-11-28T07:43:33,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742425_1601 (size=12151) 2024-11-28T07:43:33,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779873370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779873370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779873370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779873370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779873370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T07:43:33,594 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T07:43:33,667 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/967744d390ec46609f0bf35c200ef129 2024-11-28T07:43:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ffc3abe0ee4c4cd185ad96c459ccfa94 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ffc3abe0ee4c4cd185ad96c459ccfa94 2024-11-28T07:43:33,673 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ffc3abe0ee4c4cd185ad96c459ccfa94, entries=150, sequenceid=211, filesize=30.4 K 2024-11-28T07:43:33,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/4f17a36da6584758bd32ee53596e08ac as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/4f17a36da6584758bd32ee53596e08ac 2024-11-28T07:43:33,677 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/4f17a36da6584758bd32ee53596e08ac, entries=150, sequenceid=211, filesize=11.9 K 2024-11-28T07:43:33,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/967744d390ec46609f0bf35c200ef129 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/967744d390ec46609f0bf35c200ef129 2024-11-28T07:43:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,680 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/967744d390ec46609f0bf35c200ef129, entries=150, sequenceid=211, filesize=11.9 K 2024-11-28T07:43:33,681 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for a4c1644781f0b1f4c5e05c56d56502e2 in 1254ms, sequenceid=211, compaction requested=true 2024-11-28T07:43:33,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:33,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:33,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-28T07:43:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-28T07:43:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-28T07:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-28T07:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4150 sec 2024-11-28T07:43:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-28T07:43:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:33,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:33,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:33,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:33,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282de3cfa17ee64216b1f7212357a2db75_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779813883/Put/seqid=0 2024-11-28T07:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:33,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742426_1602 (size=14794) 2024-11-28T07:43:33,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779873902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779873903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779873905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779873905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:33,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779873906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:33,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779874010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779874020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779874020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779874020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779874020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779874220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779874230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779874231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779874231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779874231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,297 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:34,301 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411282de3cfa17ee64216b1f7212357a2db75_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282de3cfa17ee64216b1f7212357a2db75_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:34,301 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c668ce6465b64982932dc73df6f9fc0e, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:34,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c668ce6465b64982932dc73df6f9fc0e is 175, key is test_row_0/A:col10/1732779813883/Put/seqid=0 2024-11-28T07:43:34,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742427_1603 (size=39745) 2024-11-28T07:43:34,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-28T07:43:34,420 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:34,421 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-28T07:43:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T07:43:34,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:34,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:34,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T07:43:34,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779874530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779874540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779874540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779874541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779874550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:34,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T07:43:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:34,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:34,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c668ce6465b64982932dc73df6f9fc0e 2024-11-28T07:43:34,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/183b2a611acc4161ba57019547515098 is 50, key is test_row_0/B:col10/1732779813883/Put/seqid=0 2024-11-28T07:43:34,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742428_1604 (size=9757) 2024-11-28T07:43:34,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T07:43:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:34,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T07:43:34,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T07:43:34,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:34,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:34,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:34,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:35,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T07:43:35,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:35,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T07:43:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779875050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779875051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:35,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779875051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:35,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779875060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779875060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:35,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/183b2a611acc4161ba57019547515098 2024-11-28T07:43:35,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/78d701f20fc041d99e0aa17a64b70f6c is 50, key is test_row_0/C:col10/1732779813883/Put/seqid=0 2024-11-28T07:43:35,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742429_1605 (size=9757) 2024-11-28T07:43:35,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T07:43:35,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/78d701f20fc041d99e0aa17a64b70f6c 2024-11-28T07:43:35,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:35,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:35,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:35,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:35,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c668ce6465b64982932dc73df6f9fc0e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c668ce6465b64982932dc73df6f9fc0e 2024-11-28T07:43:35,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c668ce6465b64982932dc73df6f9fc0e, entries=200, sequenceid=226, filesize=38.8 K 2024-11-28T07:43:35,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/183b2a611acc4161ba57019547515098 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/183b2a611acc4161ba57019547515098 2024-11-28T07:43:35,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/183b2a611acc4161ba57019547515098, entries=100, sequenceid=226, filesize=9.5 K 2024-11-28T07:43:35,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/78d701f20fc041d99e0aa17a64b70f6c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/78d701f20fc041d99e0aa17a64b70f6c 2024-11-28T07:43:35,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/78d701f20fc041d99e0aa17a64b70f6c, entries=100, sequenceid=226, filesize=9.5 K 2024-11-28T07:43:35,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a4c1644781f0b1f4c5e05c56d56502e2 in 1325ms, sequenceid=226, compaction requested=true 2024-11-28T07:43:35,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:35,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:35,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:35,210 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:43:35,210 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:43:35,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:35,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:35,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:35,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:35,211 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:43:35,211 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/B is initiating minor compaction (all files) 2024-11-28T07:43:35,211 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/B in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,211 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c56e7e91b7aa4f65a2cf4829447b9d07, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/945009cc4f6e463686577968e67d866d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/4f17a36da6584758bd32ee53596e08ac, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/183b2a611acc4161ba57019547515098] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=45.5 K 2024-11-28T07:43:35,214 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133470 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:43:35,214 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/A is initiating minor compaction (all files) 2024-11-28T07:43:35,214 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/A in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,214 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/140c71e4ad0249bfadf581de3b13e03b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/826cc035aa974f969600bcab56d9d4bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ffc3abe0ee4c4cd185ad96c459ccfa94, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c668ce6465b64982932dc73df6f9fc0e] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=130.3 K 2024-11-28T07:43:35,214 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,215 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/140c71e4ad0249bfadf581de3b13e03b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/826cc035aa974f969600bcab56d9d4bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ffc3abe0ee4c4cd185ad96c459ccfa94, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c668ce6465b64982932dc73df6f9fc0e] 2024-11-28T07:43:35,215 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting c56e7e91b7aa4f65a2cf4829447b9d07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779807685 2024-11-28T07:43:35,215 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 140c71e4ad0249bfadf581de3b13e03b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779807685 2024-11-28T07:43:35,215 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 945009cc4f6e463686577968e67d866d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779808331 2024-11-28T07:43:35,216 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f17a36da6584758bd32ee53596e08ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732779810539 2024-11-28T07:43:35,216 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826cc035aa974f969600bcab56d9d4bf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779808331 2024-11-28T07:43:35,216 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 183b2a611acc4161ba57019547515098, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779813882 2024-11-28T07:43:35,216 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc3abe0ee4c4cd185ad96c459ccfa94, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732779810539 2024-11-28T07:43:35,218 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c668ce6465b64982932dc73df6f9fc0e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779812736 2024-11-28T07:43:35,238 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:35,240 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:35,240 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/1e4d425d38ad41a1b14175b896add992 is 50, key is test_row_0/B:col10/1732779813883/Put/seqid=0 2024-11-28T07:43:35,259 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128cba3d92b48a64a77971660ac85e71204_a4c1644781f0b1f4c5e05c56d56502e2 store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:35,262 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128cba3d92b48a64a77971660ac85e71204_a4c1644781f0b1f4c5e05c56d56502e2, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:35,262 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128cba3d92b48a64a77971660ac85e71204_a4c1644781f0b1f4c5e05c56d56502e2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:35,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742430_1606 (size=12697) 2024-11-28T07:43:35,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742431_1607 (size=4469) 2024-11-28T07:43:35,336 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#A#compaction#514 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:35,336 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec is 175, key is test_row_0/A:col10/1732779813883/Put/seqid=0 2024-11-28T07:43:35,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-28T07:43:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,342 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-28T07:43:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:35,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:35,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742432_1608 (size=31758) 2024-11-28T07:43:35,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a2f9ac6a08914538b5481f34d99aa9a9_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779813903/Put/seqid=0 2024-11-28T07:43:35,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742433_1609 (size=12304) 2024-11-28T07:43:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,450 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128a2f9ac6a08914538b5481f34d99aa9a9_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a2f9ac6a08914538b5481f34d99aa9a9_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:35,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/38a1876d79c64f83bae04b4f6fffe5e0, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:35,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/38a1876d79c64f83bae04b4f6fffe5e0 is 175, key is test_row_0/A:col10/1732779813903/Put/seqid=0 2024-11-28T07:43:35,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742434_1610 (size=31105) 2024-11-28T07:43:35,477 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/38a1876d79c64f83bae04b4f6fffe5e0 2024-11-28T07:43:35,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/dc52605c48054851accbe8fe69ca7267 is 50, key is test_row_0/B:col10/1732779813903/Put/seqid=0 2024-11-28T07:43:35,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742435_1611 (size=12151) 2024-11-28T07:43:35,502 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/dc52605c48054851accbe8fe69ca7267 2024-11-28T07:43:35,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/d01f52e5c8ae45fb93fcf7fb00341d07 is 50, key is test_row_0/C:col10/1732779813903/Put/seqid=0 2024-11-28T07:43:35,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742436_1612 (size=12151) 2024-11-28T07:43:35,519 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/d01f52e5c8ae45fb93fcf7fb00341d07 2024-11-28T07:43:35,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/38a1876d79c64f83bae04b4f6fffe5e0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/38a1876d79c64f83bae04b4f6fffe5e0 2024-11-28T07:43:35,530 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/38a1876d79c64f83bae04b4f6fffe5e0, entries=150, sequenceid=243, filesize=30.4 K 2024-11-28T07:43:35,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/dc52605c48054851accbe8fe69ca7267 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dc52605c48054851accbe8fe69ca7267 2024-11-28T07:43:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,534 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dc52605c48054851accbe8fe69ca7267, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T07:43:35,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/d01f52e5c8ae45fb93fcf7fb00341d07 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/d01f52e5c8ae45fb93fcf7fb00341d07 2024-11-28T07:43:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,537 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/d01f52e5c8ae45fb93fcf7fb00341d07, entries=150, sequenceid=243, filesize=11.9 K 2024-11-28T07:43:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,538 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for a4c1644781f0b1f4c5e05c56d56502e2 in 196ms, sequenceid=243, compaction requested=true 2024-11-28T07:43:35,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-28T07:43:35,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-28T07:43:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-28T07:43:35,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1160 sec 2024-11-28T07:43:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.1210 sec 2024-11-28T07:43:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-28T07:43:35,560 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,561 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-28T07:43:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T07:43:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,563 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,563 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:35,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T07:43:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-28T07:43:35,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-28T07:43:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-28T07:43:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-28T07:43:35,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-28T07:43:35,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 160 msec 2024-11-28T07:43:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,724 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/1e4d425d38ad41a1b14175b896add992 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/1e4d425d38ad41a1b14175b896add992 2024-11-28T07:43:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,729 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/B of a4c1644781f0b1f4c5e05c56d56502e2 into 1e4d425d38ad41a1b14175b896add992(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:35,730 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:35,730 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/B, priority=12, startTime=1732779815210; duration=0sec 2024-11-28T07:43:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,730 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:35,730 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:B 2024-11-28T07:43:35,730 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-28T07:43:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,731 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58771 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-28T07:43:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,731 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/C is initiating minor compaction (all files) 2024-11-28T07:43:35,731 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/C in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:35,731 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1fba934ad1fb4a88a8d2e25493f2bad1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/633d77acd63946808a8ee5d3fe51135e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/967744d390ec46609f0bf35c200ef129, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/78d701f20fc041d99e0aa17a64b70f6c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/d01f52e5c8ae45fb93fcf7fb00341d07] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=57.4 K 2024-11-28T07:43:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,732 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fba934ad1fb4a88a8d2e25493f2bad1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732779807685 2024-11-28T07:43:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,732 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 633d77acd63946808a8ee5d3fe51135e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732779808331 2024-11-28T07:43:35,732 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 967744d390ec46609f0bf35c200ef129, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732779810539 2024-11-28T07:43:35,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,733 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d701f20fc041d99e0aa17a64b70f6c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779813882 2024-11-28T07:43:35,733 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting d01f52e5c8ae45fb93fcf7fb00341d07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732779813902 2024-11-28T07:43:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,742 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,743 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/5c487e3698444f3a8764743c9ef4e565 is 50, key is test_row_0/C:col10/1732779813903/Put/seqid=0 2024-11-28T07:43:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742437_1613 (size=12731) 2024-11-28T07:43:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,795 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec 2024-11-28T07:43:35,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,800 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/A of a4c1644781f0b1f4c5e05c56d56502e2 into f60a6ff8ca0f41e7b6b9f144bb7c9eec(size=31.0 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:35,800 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:35,800 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/A, priority=12, startTime=1732779815209; duration=0sec 2024-11-28T07:43:35,800 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:35,800 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:A 2024-11-28T07:43:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-28T07:43:35,881 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,881 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-28T07:43:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T07:43:35,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,883 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,884 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:35,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T07:43:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-28T07:43:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-28T07:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-28T07:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-28T07:43:36,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 166 msec 2024-11-28T07:43:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 172 msec 2024-11-28T07:43:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:36,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-28T07:43:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:36,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:36,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:36,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:36,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:36,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112867450ed21c3144b4a5c919e0cd5faf5e_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779816101/Put/seqid=0 2024-11-28T07:43:36,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742439_1615 (size=24858) 2024-11-28T07:43:36,124 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:36,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112867450ed21c3144b4a5c919e0cd5faf5e_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112867450ed21c3144b4a5c919e0cd5faf5e_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:36,130 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/1c53b89a03394ec99e707a3c6f04a33a, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:36,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/1c53b89a03394ec99e707a3c6f04a33a is 175, key is test_row_0/A:col10/1732779816101/Put/seqid=0 2024-11-28T07:43:36,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779876129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779816119, max timestamp=1732779816134Num uniques 859; 2024-11-28T07:43:36,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:43:36,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779816119, max timestamp=1732779816134Num uniques 859; 2024-11-28T07:43:36,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:43:36,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732779816119, max timestamp=1732779816134Num uniques 859; 2024-11-28T07:43:36,139 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/f64cf8bdf19b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-28T07:43:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742438_1614 (size=74495) 2024-11-28T07:43:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779876157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779876159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779876162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779876165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,180 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/5c487e3698444f3a8764743c9ef4e565 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/5c487e3698444f3a8764743c9ef4e565 2024-11-28T07:43:36,185 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/C of a4c1644781f0b1f4c5e05c56d56502e2 into 5c487e3698444f3a8764743c9ef4e565(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:36,185 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:36,185 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/C, priority=11, startTime=1732779815210; duration=0sec 2024-11-28T07:43:36,185 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:36,185 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:C 2024-11-28T07:43:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-28T07:43:36,200 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:36,201 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-28T07:43:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-28T07:43:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T07:43:36,202 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-28T07:43:36,203 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-28T07:43:36,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-28T07:43:36,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779876250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779876270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779876270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779876280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779876280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T07:43:36,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T07:43:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779876460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779876480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779876480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779876491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,494 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779876491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T07:43:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T07:43:36,549 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=258, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/1c53b89a03394ec99e707a3c6f04a33a 2024-11-28T07:43:36,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/9c5e17c6da25430394ba8f6af26a597d is 50, key is test_row_0/B:col10/1732779816101/Put/seqid=0 2024-11-28T07:43:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742440_1616 (size=12251) 2024-11-28T07:43:36,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T07:43:36,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:36,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779876770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779876790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779876790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779876800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779876800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:36,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:36,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T07:43:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T07:43:36,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/9c5e17c6da25430394ba8f6af26a597d 2024-11-28T07:43:36,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/1ddf913cf8f94ff0b64a1b0106fd015e is 50, key is test_row_0/C:col10/1732779816101/Put/seqid=0 2024-11-28T07:43:36,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T07:43:36,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:36,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T07:43:36,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742441_1617 (size=12251) 2024-11-28T07:43:36,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/1ddf913cf8f94ff0b64a1b0106fd015e 2024-11-28T07:43:36,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/1c53b89a03394ec99e707a3c6f04a33a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/1c53b89a03394ec99e707a3c6f04a33a 2024-11-28T07:43:36,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/1c53b89a03394ec99e707a3c6f04a33a, entries=400, sequenceid=258, filesize=72.7 K 2024-11-28T07:43:36,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/9c5e17c6da25430394ba8f6af26a597d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/9c5e17c6da25430394ba8f6af26a597d 2024-11-28T07:43:36,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/9c5e17c6da25430394ba8f6af26a597d, entries=150, sequenceid=258, filesize=12.0 K 2024-11-28T07:43:36,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/1ddf913cf8f94ff0b64a1b0106fd015e as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1ddf913cf8f94ff0b64a1b0106fd015e 2024-11-28T07:43:36,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1ddf913cf8f94ff0b64a1b0106fd015e, entries=150, sequenceid=258, filesize=12.0 K 2024-11-28T07:43:36,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for a4c1644781f0b1f4c5e05c56d56502e2 in 884ms, sequenceid=258, compaction requested=true 2024-11-28T07:43:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:36,986 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:36,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:36,986 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:36,987 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137358 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:36,987 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:36,987 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/A is initiating minor compaction (all files) 2024-11-28T07:43:36,987 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/B is initiating minor compaction (all files) 2024-11-28T07:43:36,987 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/A in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,987 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/B in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,987 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/38a1876d79c64f83bae04b4f6fffe5e0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/1c53b89a03394ec99e707a3c6f04a33a] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=134.1 K 2024-11-28T07:43:36,987 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/1e4d425d38ad41a1b14175b896add992, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dc52605c48054851accbe8fe69ca7267, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/9c5e17c6da25430394ba8f6af26a597d] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=36.2 K 2024-11-28T07:43:36,987 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:36,987 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/38a1876d79c64f83bae04b4f6fffe5e0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/1c53b89a03394ec99e707a3c6f04a33a] 2024-11-28T07:43:36,987 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4d425d38ad41a1b14175b896add992, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779810539 2024-11-28T07:43:36,987 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60a6ff8ca0f41e7b6b9f144bb7c9eec, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732779810539 2024-11-28T07:43:36,988 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting dc52605c48054851accbe8fe69ca7267, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732779813902 2024-11-28T07:43:36,988 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a1876d79c64f83bae04b4f6fffe5e0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732779813902 2024-11-28T07:43:36,988 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c5e17c6da25430394ba8f6af26a597d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732779816095 2024-11-28T07:43:36,988 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c53b89a03394ec99e707a3c6f04a33a, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732779816090 2024-11-28T07:43:36,998 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:37,000 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#B#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:37,000 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112803c87a35c256447b8a4ee2b911b7d1ce_a4c1644781f0b1f4c5e05c56d56502e2 store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:37,000 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/5b5eeff243f7445f907178c5b9a917f2 is 50, key is test_row_0/B:col10/1732779816101/Put/seqid=0 2024-11-28T07:43:37,004 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112803c87a35c256447b8a4ee2b911b7d1ce_a4c1644781f0b1f4c5e05c56d56502e2, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:37,004 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112803c87a35c256447b8a4ee2b911b7d1ce_a4c1644781f0b1f4c5e05c56d56502e2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:37,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742442_1618 (size=4469) 2024-11-28T07:43:37,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742443_1619 (size=12899) 2024-11-28T07:43:37,026 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/5b5eeff243f7445f907178c5b9a917f2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5b5eeff243f7445f907178c5b9a917f2 2024-11-28T07:43:37,033 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/B of a4c1644781f0b1f4c5e05c56d56502e2 into 5b5eeff243f7445f907178c5b9a917f2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:37,033 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:37,033 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/B, priority=13, startTime=1732779816986; duration=0sec 2024-11-28T07:43:37,033 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:37,033 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:B 2024-11-28T07:43:37,033 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-28T07:43:37,033 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-28T07:43:37,034 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-28T07:43:37,034 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. because compaction request was cancelled 2024-11-28T07:43:37,034 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:C 2024-11-28T07:43:37,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34437 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-28T07:43:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:37,119 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-28T07:43:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:37,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-28T07:43:37,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bbc03ef8aa8e4482a83585b317e78b1a_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779816156/Put/seqid=0 2024-11-28T07:43:37,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742444_1620 (size=12454) 2024-11-28T07:43:37,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:37,133 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bbc03ef8aa8e4482a83585b317e78b1a_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bbc03ef8aa8e4482a83585b317e78b1a_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/bf62094df89548f0baff38f3d3a660c5, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:37,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/bf62094df89548f0baff38f3d3a660c5 is 175, key is test_row_0/A:col10/1732779816156/Put/seqid=0 2024-11-28T07:43:37,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742445_1621 (size=31255) 2024-11-28T07:43:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:37,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. as already flushing 2024-11-28T07:43:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779877304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779877305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779877307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779877310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779877310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T07:43:37,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779877410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,417 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#A#compaction#523 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:37,418 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c7524976fc434a0a9673964e035427c8 is 175, key is test_row_0/A:col10/1732779816101/Put/seqid=0 2024-11-28T07:43:37,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742446_1622 (size=31853) 2024-11-28T07:43:37,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779877420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779877420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,509 INFO [Thread-2438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2438. 2024-11-28T07:43:37,509 DEBUG [Thread-2438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:37,509 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,509 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:37,510 INFO [Thread-2432 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2432. 2024-11-28T07:43:37,510 DEBUG [Thread-2432 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:37,510 DEBUG [Thread-2432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,510 DEBUG [Thread-2432 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:37,512 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-11-28T07:43:37,512 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:37,512 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,512 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,512 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:37,514 INFO [Thread-2434 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2434. 2024-11-28T07:43:37,514 INFO [Thread-2436 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2436. 2024-11-28T07:43:37,514 DEBUG [Thread-2434 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:37,514 DEBUG [Thread-2436 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:37,514 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,514 DEBUG [Thread-2436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,514 DEBUG [Thread-2436 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,514 DEBUG [Thread-2434 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:37,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:37,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:37,538 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/bf62094df89548f0baff38f3d3a660c5 2024-11-28T07:43:37,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/a0f94d0738384c7b8b0848934e16eb6a is 50, key is test_row_0/B:col10/1732779816156/Put/seqid=0 2024-11-28T07:43:37,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742447_1623 (size=12301) 2024-11-28T07:43:37,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779877620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779877630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779877630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,825 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/c7524976fc434a0a9673964e035427c8 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c7524976fc434a0a9673964e035427c8 2024-11-28T07:43:37,828 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/A of a4c1644781f0b1f4c5e05c56d56502e2 into c7524976fc434a0a9673964e035427c8(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:37,828 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:37,828 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/A, priority=13, startTime=1732779816986; duration=0sec 2024-11-28T07:43:37,828 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:37,828 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:A 2024-11-28T07:43:37,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36572 deadline: 1732779877930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36540 deadline: 1732779877940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36562 deadline: 1732779877940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:37,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/a0f94d0738384c7b8b0848934e16eb6a 2024-11-28T07:43:37,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/2344ace3871f4c3ab9e07749590e71f3 is 50, key is test_row_0/C:col10/1732779816156/Put/seqid=0 2024-11-28T07:43:37,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742448_1624 (size=12301) 2024-11-28T07:43:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-28T07:43:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36598 deadline: 1732779878330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36566 deadline: 1732779878330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:38,331 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_0' in a4c1644781f0b1f4c5e05c56d56502e2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:43:38,331 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f64cf8bdf19b,34437,1732779635619 for 'test_row_2' in a4c1644781f0b1f4c5e05c56d56502e2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-28T07:43:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 , the old value is region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-28T07:43:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., hostname=f64cf8bdf19b,34437,1732779635619, seqNum=5 because the exception is null or not the one we care about 2024-11-28T07:43:38,356 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/2344ace3871f4c3ab9e07749590e71f3 2024-11-28T07:43:38,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/bf62094df89548f0baff38f3d3a660c5 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/bf62094df89548f0baff38f3d3a660c5 2024-11-28T07:43:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T07:43:38,362 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/bf62094df89548f0baff38f3d3a660c5, entries=150, sequenceid=295, filesize=30.5 K 2024-11-28T07:43:38,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/a0f94d0738384c7b8b0848934e16eb6a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/a0f94d0738384c7b8b0848934e16eb6a 2024-11-28T07:43:38,365 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/a0f94d0738384c7b8b0848934e16eb6a, entries=150, sequenceid=295, filesize=12.0 K 2024-11-28T07:43:38,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/2344ace3871f4c3ab9e07749590e71f3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2344ace3871f4c3ab9e07749590e71f3 2024-11-28T07:43:38,368 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2344ace3871f4c3ab9e07749590e71f3, entries=150, sequenceid=295, filesize=12.0 K 2024-11-28T07:43:38,369 INFO [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for a4c1644781f0b1f4c5e05c56d56502e2 in 1249ms, sequenceid=295, compaction requested=true 2024-11-28T07:43:38,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:38,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:38,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f64cf8bdf19b:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-28T07:43:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-28T07:43:38,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-28T07:43:38,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1670 sec 2024-11-28T07:43:38,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.1700 sec 2024-11-28T07:43:38,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34437 {}] regionserver.HRegion(8855): Flush requested on a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:38,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-28T07:43:38,441 INFO [Thread-2425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2425. 2024-11-28T07:43:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:38,441 DEBUG [Thread-2425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:38,441 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:38,442 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:38,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:38,442 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:38,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f8b69d2eae99442ebdc058a36578f516_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779817302/Put/seqid=0 2024-11-28T07:43:38,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742449_1625 (size=12454) 2024-11-28T07:43:38,451 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-28T07:43:38,451 INFO [Thread-2421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2421. 2024-11-28T07:43:38,451 DEBUG [Thread-2421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:38,451 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:38,451 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:38,451 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:38,452 DEBUG [Thread-2421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:38,452 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:38,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:38,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:38,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:38,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128f8b69d2eae99442ebdc058a36578f516_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f8b69d2eae99442ebdc058a36578f516_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:38,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ad43e1140df3472d854bb4b70f2183a1, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:38,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ad43e1140df3472d854bb4b70f2183a1 is 175, key is test_row_0/A:col10/1732779817302/Put/seqid=0 2024-11-28T07:43:38,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742450_1626 (size=31255) 2024-11-28T07:43:39,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ad43e1140df3472d854bb4b70f2183a1 2024-11-28T07:43:39,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/65895c0a858e4586a766087504d376a0 is 50, key is test_row_0/B:col10/1732779817302/Put/seqid=0 2024-11-28T07:43:39,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742451_1627 (size=12301) 2024-11-28T07:43:39,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/65895c0a858e4586a766087504d376a0 2024-11-28T07:43:39,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/36f82180e5fb47868e0c0a88d6218987 is 50, key is test_row_0/C:col10/1732779817302/Put/seqid=0 2024-11-28T07:43:39,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742452_1628 (size=12301) 2024-11-28T07:43:40,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/36f82180e5fb47868e0c0a88d6218987 2024-11-28T07:43:40,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/ad43e1140df3472d854bb4b70f2183a1 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ad43e1140df3472d854bb4b70f2183a1 2024-11-28T07:43:40,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ad43e1140df3472d854bb4b70f2183a1, entries=150, sequenceid=308, filesize=30.5 K 2024-11-28T07:43:40,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/65895c0a858e4586a766087504d376a0 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/65895c0a858e4586a766087504d376a0 2024-11-28T07:43:40,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/65895c0a858e4586a766087504d376a0, entries=150, sequenceid=308, filesize=12.0 K 2024-11-28T07:43:40,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/36f82180e5fb47868e0c0a88d6218987 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/36f82180e5fb47868e0c0a88d6218987 2024-11-28T07:43:40,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/36f82180e5fb47868e0c0a88d6218987, entries=150, sequenceid=308, filesize=12.0 K 2024-11-28T07:43:40,086 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=13.42 KB/13740 for a4c1644781f0b1f4c5e05c56d56502e2 in 1645ms, sequenceid=308, compaction requested=true 2024-11-28T07:43:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:A, priority=-2147483648, current under compaction store size is 1 2024-11-28T07:43:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:40,086 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:B, priority=-2147483648, current under compaction store size is 2 2024-11-28T07:43:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a4c1644781f0b1f4c5e05c56d56502e2:C, priority=-2147483648, current under compaction store size is 3 2024-11-28T07:43:40,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:40,086 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/A is initiating minor compaction (all files) 2024-11-28T07:43:40,087 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/A in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/B is initiating minor compaction (all files) 2024-11-28T07:43:40,087 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c7524976fc434a0a9673964e035427c8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/bf62094df89548f0baff38f3d3a660c5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ad43e1140df3472d854bb4b70f2183a1] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=92.2 K 2024-11-28T07:43:40,087 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/B in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,087 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,087 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5b5eeff243f7445f907178c5b9a917f2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/a0f94d0738384c7b8b0848934e16eb6a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/65895c0a858e4586a766087504d376a0] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=36.6 K 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. files: [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c7524976fc434a0a9673964e035427c8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/bf62094df89548f0baff38f3d3a660c5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ad43e1140df3472d854bb4b70f2183a1] 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b5eeff243f7445f907178c5b9a917f2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732779816095 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7524976fc434a0a9673964e035427c8, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732779816095 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf62094df89548f0baff38f3d3a660c5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732779816156 2024-11-28T07:43:40,087 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting a0f94d0738384c7b8b0848934e16eb6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732779816156 2024-11-28T07:43:40,088 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 65895c0a858e4586a766087504d376a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732779817280 2024-11-28T07:43:40,088 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad43e1140df3472d854bb4b70f2183a1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732779817280 2024-11-28T07:43:40,093 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:40,094 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#B#compaction#530 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:40,094 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/dd4437c7af654fe9918f475cdadc456d is 50, key is test_row_0/B:col10/1732779817302/Put/seqid=0 2024-11-28T07:43:40,095 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241128bf31b67d0e264d99905f6b56abe56fe7_a4c1644781f0b1f4c5e05c56d56502e2 store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:40,097 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241128bf31b67d0e264d99905f6b56abe56fe7_a4c1644781f0b1f4c5e05c56d56502e2, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:40,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742453_1629 (size=13051) 2024-11-28T07:43:40,097 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241128bf31b67d0e264d99905f6b56abe56fe7_a4c1644781f0b1f4c5e05c56d56502e2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:40,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742454_1630 (size=4469) 2024-11-28T07:43:40,342 INFO [Thread-2423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2423. 2024-11-28T07:43:40,342 DEBUG [Thread-2423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:40,342 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:40,342 DEBUG [Thread-2423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:40,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:40,351 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-28T07:43:40,351 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-28T07:43:40,351 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:40,351 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:40,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-28T07:43:40,370 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3358 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3204 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3212 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3371 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3206 2024-11-28T07:43:40,370 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-28T07:43:40,370 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:43:40,370 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:43:40,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:40,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:40,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:40,371 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-28T07:43:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T07:43:40,373 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779820373"}]},"ts":"1732779820373"} 2024-11-28T07:43:40,374 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-28T07:43:40,374 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-28T07:43:40,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-28T07:43:40,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, UNASSIGN}] 2024-11-28T07:43:40,377 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, UNASSIGN 2024-11-28T07:43:40,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=CLOSING, regionLocation=f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:40,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, UNASSIGN because future has completed 2024-11-28T07:43:40,378 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-28T07:43:40,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619}] 2024-11-28T07:43:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T07:43:40,500 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#A#compaction#531 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:40,501 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/dd4437c7af654fe9918f475cdadc456d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dd4437c7af654fe9918f475cdadc456d 2024-11-28T07:43:40,501 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/102212e4f0cc46e9830d3bb2de77d4b3 is 175, key is test_row_0/A:col10/1732779817302/Put/seqid=0 2024-11-28T07:43:40,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742455_1631 (size=32005) 2024-11-28T07:43:40,504 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/B of a4c1644781f0b1f4c5e05c56d56502e2 into dd4437c7af654fe9918f475cdadc456d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:40,504 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:40,504 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/B, priority=13, startTime=1732779820086; duration=0sec 2024-11-28T07:43:40,504 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-28T07:43:40,504 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:B 2024-11-28T07:43:40,504 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-28T07:43:40,505 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49584 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-28T07:43:40,505 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1541): a4c1644781f0b1f4c5e05c56d56502e2/C is initiating minor compaction (all files) 2024-11-28T07:43:40,505 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a4c1644781f0b1f4c5e05c56d56502e2/C in TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,505 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/5c487e3698444f3a8764743c9ef4e565, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1ddf913cf8f94ff0b64a1b0106fd015e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2344ace3871f4c3ab9e07749590e71f3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/36f82180e5fb47868e0c0a88d6218987] into tmpdir=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp, totalSize=48.4 K 2024-11-28T07:43:40,506 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c487e3698444f3a8764743c9ef4e565, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1732779813902 2024-11-28T07:43:40,506 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ddf913cf8f94ff0b64a1b0106fd015e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732779816095 2024-11-28T07:43:40,506 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 2344ace3871f4c3ab9e07749590e71f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732779816156 2024-11-28T07:43:40,506 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] compactions.Compactor(225): Compacting 36f82180e5fb47868e0c0a88d6218987, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732779817280 2024-11-28T07:43:40,512 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a4c1644781f0b1f4c5e05c56d56502e2#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-28T07:43:40,513 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/b0f51d8f75c4443580e06d540d188ff2 is 50, key is test_row_0/C:col10/1732779817302/Put/seqid=0 2024-11-28T07:43:40,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742456_1632 (size=13017) 2024-11-28T07:43:40,530 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:40,530 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-28T07:43:40,530 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing a4c1644781f0b1f4c5e05c56d56502e2, disabling compactions & flushes 2024-11-28T07:43:40,530 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T07:43:40,908 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/102212e4f0cc46e9830d3bb2de77d4b3 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/102212e4f0cc46e9830d3bb2de77d4b3 2024-11-28T07:43:40,911 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/A of a4c1644781f0b1f4c5e05c56d56502e2 into 102212e4f0cc46e9830d3bb2de77d4b3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:40,911 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:40,911 INFO [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/A, priority=13, startTime=1732779820086; duration=0sec 2024-11-28T07:43:40,911 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:40,911 DEBUG [RS:0;f64cf8bdf19b:34437-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:A 2024-11-28T07:43:40,919 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/b0f51d8f75c4443580e06d540d188ff2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/b0f51d8f75c4443580e06d540d188ff2 2024-11-28T07:43:40,922 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a4c1644781f0b1f4c5e05c56d56502e2/C of a4c1644781f0b1f4c5e05c56d56502e2 into b0f51d8f75c4443580e06d540d188ff2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-28T07:43:40,922 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a4c1644781f0b1f4c5e05c56d56502e2: 2024-11-28T07:43:40,922 INFO [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., storeName=a4c1644781f0b1f4c5e05c56d56502e2/C, priority=12, startTime=1732779820086; duration=0sec 2024-11-28T07:43:40,922 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,922 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,922 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-28T07:43:40,922 DEBUG [RS:0;f64cf8bdf19b:34437-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a4c1644781f0b1f4c5e05c56d56502e2:C 2024-11-28T07:43:40,922 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. after waiting 0 ms 2024-11-28T07:43:40,922 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:40,922 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing a4c1644781f0b1f4c5e05c56d56502e2 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-28T07:43:40,922 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=A 2024-11-28T07:43:40,923 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:40,923 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=B 2024-11-28T07:43:40,923 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:40,923 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a4c1644781f0b1f4c5e05c56d56502e2, store=C 2024-11-28T07:43:40,923 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-28T07:43:40,927 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112830e7b40158e1424fb29c07c525532438_a4c1644781f0b1f4c5e05c56d56502e2 is 50, key is test_row_0/A:col10/1732779820350/Put/seqid=0 2024-11-28T07:43:40,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742457_1633 (size=12454) 2024-11-28T07:43:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T07:43:41,330 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T07:43:41,333 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112830e7b40158e1424fb29c07c525532438_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112830e7b40158e1424fb29c07c525532438_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:41,334 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d2f35abaf31c4843b60e386826673034, store: [table=TestAcidGuarantees family=A region=a4c1644781f0b1f4c5e05c56d56502e2] 2024-11-28T07:43:41,334 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d2f35abaf31c4843b60e386826673034 is 175, key is test_row_0/A:col10/1732779820350/Put/seqid=0 2024-11-28T07:43:41,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742458_1634 (size=31255) 2024-11-28T07:43:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T07:43:41,738 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d2f35abaf31c4843b60e386826673034 2024-11-28T07:43:41,743 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/5195c97186874c12865019cf6aecb41d is 50, key is test_row_0/B:col10/1732779820350/Put/seqid=0 2024-11-28T07:43:41,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742459_1635 (size=12301) 2024-11-28T07:43:42,146 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/5195c97186874c12865019cf6aecb41d 2024-11-28T07:43:42,151 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/a7ca6110ef214fa38a529ec4e596af25 is 50, key is test_row_0/C:col10/1732779820350/Put/seqid=0 2024-11-28T07:43:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742460_1636 (size=12301) 2024-11-28T07:43:42,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T07:43:42,554 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/a7ca6110ef214fa38a529ec4e596af25 2024-11-28T07:43:42,557 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/A/d2f35abaf31c4843b60e386826673034 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d2f35abaf31c4843b60e386826673034 2024-11-28T07:43:42,560 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d2f35abaf31c4843b60e386826673034, entries=150, sequenceid=318, filesize=30.5 K 2024-11-28T07:43:42,560 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/B/5195c97186874c12865019cf6aecb41d as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5195c97186874c12865019cf6aecb41d 2024-11-28T07:43:42,563 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5195c97186874c12865019cf6aecb41d, entries=150, sequenceid=318, filesize=12.0 K 2024-11-28T07:43:42,564 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/.tmp/C/a7ca6110ef214fa38a529ec4e596af25 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/a7ca6110ef214fa38a529ec4e596af25 2024-11-28T07:43:42,566 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/a7ca6110ef214fa38a529ec4e596af25, entries=150, sequenceid=318, filesize=12.0 K 2024-11-28T07:43:42,567 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a4c1644781f0b1f4c5e05c56d56502e2 in 1644ms, sequenceid=318, compaction requested=false 2024-11-28T07:43:42,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/2006bf362c8841138c6d98053a84828b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/a407e18832664ea6bd88264030e6a2a6, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/cb37bb49105946c699a00b983fc41bc4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/568ec2d30c8449d6a999548dd64edc95, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d57a1486b3fc45ef99d266e9af138234, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e013d474b7944697a59054c60915ec71, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c203b35e12e94d1591bca41a449ec1c4, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e91f045ccd7c4ebfbda3dd536d5b0439, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/16b25389764e48d6bea098cce6deff66, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/8ec91562be6949fe91d17b65134a2928, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e4e87b5cd9c94f9db319d94009d114f8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/140c71e4ad0249bfadf581de3b13e03b, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/195386d9c9ba42798ace9f955c22c3da, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/826cc035aa974f969600bcab56d9d4bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ffc3abe0ee4c4cd185ad96c459ccfa94, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c668ce6465b64982932dc73df6f9fc0e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/38a1876d79c64f83bae04b4f6fffe5e0, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/1c53b89a03394ec99e707a3c6f04a33a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c7524976fc434a0a9673964e035427c8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/bf62094df89548f0baff38f3d3a660c5, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ad43e1140df3472d854bb4b70f2183a1] to archive 2024-11-28T07:43:42,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:43:42,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/2006bf362c8841138c6d98053a84828b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/2006bf362c8841138c6d98053a84828b 2024-11-28T07:43:42,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/a407e18832664ea6bd88264030e6a2a6 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/a407e18832664ea6bd88264030e6a2a6 2024-11-28T07:43:42,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/cb37bb49105946c699a00b983fc41bc4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/cb37bb49105946c699a00b983fc41bc4 2024-11-28T07:43:42,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/568ec2d30c8449d6a999548dd64edc95 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/568ec2d30c8449d6a999548dd64edc95 2024-11-28T07:43:42,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d57a1486b3fc45ef99d266e9af138234 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d57a1486b3fc45ef99d266e9af138234 2024-11-28T07:43:42,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e013d474b7944697a59054c60915ec71 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e013d474b7944697a59054c60915ec71 2024-11-28T07:43:42,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c203b35e12e94d1591bca41a449ec1c4 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c203b35e12e94d1591bca41a449ec1c4 2024-11-28T07:43:42,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e91f045ccd7c4ebfbda3dd536d5b0439 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e91f045ccd7c4ebfbda3dd536d5b0439 2024-11-28T07:43:42,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/16b25389764e48d6bea098cce6deff66 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/16b25389764e48d6bea098cce6deff66 2024-11-28T07:43:42,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/8ec91562be6949fe91d17b65134a2928 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/8ec91562be6949fe91d17b65134a2928 2024-11-28T07:43:42,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e4e87b5cd9c94f9db319d94009d114f8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/e4e87b5cd9c94f9db319d94009d114f8 2024-11-28T07:43:42,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/140c71e4ad0249bfadf581de3b13e03b to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/140c71e4ad0249bfadf581de3b13e03b 2024-11-28T07:43:42,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/195386d9c9ba42798ace9f955c22c3da to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/195386d9c9ba42798ace9f955c22c3da 2024-11-28T07:43:42,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/826cc035aa974f969600bcab56d9d4bf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/826cc035aa974f969600bcab56d9d4bf 2024-11-28T07:43:42,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ffc3abe0ee4c4cd185ad96c459ccfa94 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ffc3abe0ee4c4cd185ad96c459ccfa94 2024-11-28T07:43:42,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c668ce6465b64982932dc73df6f9fc0e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c668ce6465b64982932dc73df6f9fc0e 2024-11-28T07:43:42,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/f60a6ff8ca0f41e7b6b9f144bb7c9eec 2024-11-28T07:43:42,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/38a1876d79c64f83bae04b4f6fffe5e0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/38a1876d79c64f83bae04b4f6fffe5e0 2024-11-28T07:43:42,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/1c53b89a03394ec99e707a3c6f04a33a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/1c53b89a03394ec99e707a3c6f04a33a 2024-11-28T07:43:42,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c7524976fc434a0a9673964e035427c8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/c7524976fc434a0a9673964e035427c8 2024-11-28T07:43:42,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/bf62094df89548f0baff38f3d3a660c5 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/bf62094df89548f0baff38f3d3a660c5 2024-11-28T07:43:42,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ad43e1140df3472d854bb4b70f2183a1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/ad43e1140df3472d854bb4b70f2183a1 2024-11-28T07:43:42,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/207810e9107d49dda8e3c16f9d293e3c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d999bf712b98469790e09340146ecff3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/3482875adfec4c2689126c4d947601bc, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d749c6b2c2bd49fc9a089248dd64719d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/6892bba265ce45eca67aa3e001bcb00e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/91ab40d28adb45a7bfc8387cb7d6c0db, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c124b43a1baa44149156875eb80a169c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/bf99ccb332084f49bb3dea8e2be1f64a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/f701f182a2604ec79f8de90ad07763aa, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/2a52e709f8c94c04b4031b68d566c343, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c56e7e91b7aa4f65a2cf4829447b9d07, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/71aa1299dbcb4be48e8e10bb19511375, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/945009cc4f6e463686577968e67d866d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/4f17a36da6584758bd32ee53596e08ac, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/1e4d425d38ad41a1b14175b896add992, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/183b2a611acc4161ba57019547515098, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dc52605c48054851accbe8fe69ca7267, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5b5eeff243f7445f907178c5b9a917f2, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/9c5e17c6da25430394ba8f6af26a597d, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/a0f94d0738384c7b8b0848934e16eb6a, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/65895c0a858e4586a766087504d376a0] to archive 2024-11-28T07:43:42,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:43:42,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/207810e9107d49dda8e3c16f9d293e3c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/207810e9107d49dda8e3c16f9d293e3c 2024-11-28T07:43:42,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d999bf712b98469790e09340146ecff3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d999bf712b98469790e09340146ecff3 2024-11-28T07:43:42,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/3482875adfec4c2689126c4d947601bc to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/3482875adfec4c2689126c4d947601bc 2024-11-28T07:43:42,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d749c6b2c2bd49fc9a089248dd64719d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/d749c6b2c2bd49fc9a089248dd64719d 2024-11-28T07:43:42,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/6892bba265ce45eca67aa3e001bcb00e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/6892bba265ce45eca67aa3e001bcb00e 2024-11-28T07:43:42,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/91ab40d28adb45a7bfc8387cb7d6c0db to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/91ab40d28adb45a7bfc8387cb7d6c0db 2024-11-28T07:43:42,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c124b43a1baa44149156875eb80a169c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c124b43a1baa44149156875eb80a169c 2024-11-28T07:43:42,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/7fc7642dd8bc4ce1b1e5f534ea7a15cf 2024-11-28T07:43:42,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/bf99ccb332084f49bb3dea8e2be1f64a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/bf99ccb332084f49bb3dea8e2be1f64a 2024-11-28T07:43:42,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/f701f182a2604ec79f8de90ad07763aa to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/f701f182a2604ec79f8de90ad07763aa 2024-11-28T07:43:42,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/2a52e709f8c94c04b4031b68d566c343 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/2a52e709f8c94c04b4031b68d566c343 2024-11-28T07:43:42,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c56e7e91b7aa4f65a2cf4829447b9d07 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/c56e7e91b7aa4f65a2cf4829447b9d07 2024-11-28T07:43:42,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/71aa1299dbcb4be48e8e10bb19511375 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/71aa1299dbcb4be48e8e10bb19511375 2024-11-28T07:43:42,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/945009cc4f6e463686577968e67d866d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/945009cc4f6e463686577968e67d866d 2024-11-28T07:43:42,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/4f17a36da6584758bd32ee53596e08ac to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/4f17a36da6584758bd32ee53596e08ac 2024-11-28T07:43:42,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/1e4d425d38ad41a1b14175b896add992 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/1e4d425d38ad41a1b14175b896add992 2024-11-28T07:43:42,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/183b2a611acc4161ba57019547515098 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/183b2a611acc4161ba57019547515098 2024-11-28T07:43:42,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dc52605c48054851accbe8fe69ca7267 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dc52605c48054851accbe8fe69ca7267 2024-11-28T07:43:42,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5b5eeff243f7445f907178c5b9a917f2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5b5eeff243f7445f907178c5b9a917f2 2024-11-28T07:43:42,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/9c5e17c6da25430394ba8f6af26a597d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/9c5e17c6da25430394ba8f6af26a597d 2024-11-28T07:43:42,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/a0f94d0738384c7b8b0848934e16eb6a to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/a0f94d0738384c7b8b0848934e16eb6a 2024-11-28T07:43:42,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/65895c0a858e4586a766087504d376a0 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/65895c0a858e4586a766087504d376a0 2024-11-28T07:43:42,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/0a3be63851584edaba765494686463bf, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/83305daca0d44214b80ce78169adf3d1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/3befffa778234cbaaa26bb5ba4976ad8, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/cacbc707620c48c895e3fdbb59112c03, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/30d73b6adca145e4a3f53e7d89e06e52, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2fc784aaa64943ebbd69eb66a7411fdd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/8c4c1cb0d9e042f1b8ac85ffb7005856, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/4ac8a0294e384e66ba83b209ed3a5292, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/9c3a26dc13cd4bff9c908cf24fb8b289, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/25213a27c60a48d3a9f1d1ae98035e21, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/69fee21b2bb243649d004b1b6ca44164, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1fba934ad1fb4a88a8d2e25493f2bad1, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/bd3037de70c6472387083625c18cb2dd, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/633d77acd63946808a8ee5d3fe51135e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/967744d390ec46609f0bf35c200ef129, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/78d701f20fc041d99e0aa17a64b70f6c, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/5c487e3698444f3a8764743c9ef4e565, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/d01f52e5c8ae45fb93fcf7fb00341d07, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1ddf913cf8f94ff0b64a1b0106fd015e, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2344ace3871f4c3ab9e07749590e71f3, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/36f82180e5fb47868e0c0a88d6218987] to archive 2024-11-28T07:43:42,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-28T07:43:42,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/0a3be63851584edaba765494686463bf to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/0a3be63851584edaba765494686463bf 2024-11-28T07:43:42,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/83305daca0d44214b80ce78169adf3d1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/83305daca0d44214b80ce78169adf3d1 2024-11-28T07:43:42,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/3befffa778234cbaaa26bb5ba4976ad8 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/3befffa778234cbaaa26bb5ba4976ad8 2024-11-28T07:43:42,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/cacbc707620c48c895e3fdbb59112c03 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/cacbc707620c48c895e3fdbb59112c03 2024-11-28T07:43:42,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/30d73b6adca145e4a3f53e7d89e06e52 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/30d73b6adca145e4a3f53e7d89e06e52 2024-11-28T07:43:42,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2fc784aaa64943ebbd69eb66a7411fdd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2fc784aaa64943ebbd69eb66a7411fdd 2024-11-28T07:43:42,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/8c4c1cb0d9e042f1b8ac85ffb7005856 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/8c4c1cb0d9e042f1b8ac85ffb7005856 2024-11-28T07:43:42,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/4ac8a0294e384e66ba83b209ed3a5292 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/4ac8a0294e384e66ba83b209ed3a5292 2024-11-28T07:43:42,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/9c3a26dc13cd4bff9c908cf24fb8b289 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/9c3a26dc13cd4bff9c908cf24fb8b289 2024-11-28T07:43:42,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/25213a27c60a48d3a9f1d1ae98035e21 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/25213a27c60a48d3a9f1d1ae98035e21 2024-11-28T07:43:42,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/69fee21b2bb243649d004b1b6ca44164 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/69fee21b2bb243649d004b1b6ca44164 2024-11-28T07:43:42,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1fba934ad1fb4a88a8d2e25493f2bad1 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1fba934ad1fb4a88a8d2e25493f2bad1 2024-11-28T07:43:42,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/bd3037de70c6472387083625c18cb2dd to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/bd3037de70c6472387083625c18cb2dd 2024-11-28T07:43:42,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/633d77acd63946808a8ee5d3fe51135e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/633d77acd63946808a8ee5d3fe51135e 2024-11-28T07:43:42,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/967744d390ec46609f0bf35c200ef129 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/967744d390ec46609f0bf35c200ef129 2024-11-28T07:43:42,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/78d701f20fc041d99e0aa17a64b70f6c to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/78d701f20fc041d99e0aa17a64b70f6c 2024-11-28T07:43:42,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/5c487e3698444f3a8764743c9ef4e565 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/5c487e3698444f3a8764743c9ef4e565 2024-11-28T07:43:42,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/d01f52e5c8ae45fb93fcf7fb00341d07 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/d01f52e5c8ae45fb93fcf7fb00341d07 2024-11-28T07:43:42,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1ddf913cf8f94ff0b64a1b0106fd015e to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/1ddf913cf8f94ff0b64a1b0106fd015e 2024-11-28T07:43:42,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2344ace3871f4c3ab9e07749590e71f3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/2344ace3871f4c3ab9e07749590e71f3 2024-11-28T07:43:42,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/36f82180e5fb47868e0c0a88d6218987 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/36f82180e5fb47868e0c0a88d6218987 2024-11-28T07:43:42,623 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/recovered.edits/321.seqid, newMaxSeqId=321, maxSeqId=4 2024-11-28T07:43:42,623 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. 2024-11-28T07:43:42,623 DEBUG [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for a4c1644781f0b1f4c5e05c56d56502e2: Waiting for close lock at 1732779820530Running coprocessor pre-close hooks at 1732779820530Disabling compacts and flushes for region at 1732779820530Disabling writes for close at 1732779820922 (+392 ms)Obtaining lock to block concurrent updates at 1732779820922Preparing flush snapshotting stores in a4c1644781f0b1f4c5e05c56d56502e2 at 1732779820922Finished memstore snapshotting TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732779820923 (+1 ms)Flushing stores of TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2. at 1732779820923Flushing a4c1644781f0b1f4c5e05c56d56502e2/A: creating writer at 1732779820923Flushing mob file a4c1644781f0b1f4c5e05c56d56502e2/A: appending metadata at 1732779820927 (+4 ms)Flushing mob file a4c1644781f0b1f4c5e05c56d56502e2/A: closing flushed file at 1732779820927Flushing a4c1644781f0b1f4c5e05c56d56502e2/A: appending metadata at 1732779821334 (+407 ms)Flushing a4c1644781f0b1f4c5e05c56d56502e2/A: closing flushed file at 1732779821334Flushing a4c1644781f0b1f4c5e05c56d56502e2/B: creating writer at 1732779821740 (+406 ms)Flushing a4c1644781f0b1f4c5e05c56d56502e2/B: appending metadata at 1732779821742 (+2 ms)Flushing a4c1644781f0b1f4c5e05c56d56502e2/B: closing flushed file at 1732779821742Flushing a4c1644781f0b1f4c5e05c56d56502e2/C: creating writer at 1732779822148 (+406 ms)Flushing a4c1644781f0b1f4c5e05c56d56502e2/C: appending metadata at 1732779822150 (+2 ms)Flushing a4c1644781f0b1f4c5e05c56d56502e2/C: closing flushed file at 1732779822150Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4720db58: reopening flushed file at 1732779822557 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16e33574: reopening flushed file at 1732779822560 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c037168: reopening flushed file at 1732779822563 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for a4c1644781f0b1f4c5e05c56d56502e2 in 1644ms, sequenceid=318, compaction requested=false at 1732779822567 (+4 ms)Writing region close event to WAL at 1732779822620 (+53 ms)Running coprocessor post-close hooks at 1732779822623 (+3 ms)Closed at 1732779822623 2024-11-28T07:43:42,625 INFO [RS_CLOSE_REGION-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:42,625 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=a4c1644781f0b1f4c5e05c56d56502e2, regionState=CLOSED 2024-11-28T07:43:42,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 because future has completed 2024-11-28T07:43:42,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-28T07:43:42,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure a4c1644781f0b1f4c5e05c56d56502e2, server=f64cf8bdf19b,34437,1732779635619 in 2.2490 sec 2024-11-28T07:43:42,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-28T07:43:42,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a4c1644781f0b1f4c5e05c56d56502e2, UNASSIGN in 2.2520 sec 2024-11-28T07:43:42,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-28T07:43:42,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2550 sec 2024-11-28T07:43:42,632 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732779822632"}]},"ts":"1732779822632"} 2024-11-28T07:43:42,633 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-28T07:43:42,633 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-28T07:43:42,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2630 sec 2024-11-28T07:43:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-28T07:43:44,541 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:44,541 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-28T07:43:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:44,543 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T07:43:44,543 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:44,546 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,547 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C, FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/recovered.edits] 2024-11-28T07:43:44,549 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/102212e4f0cc46e9830d3bb2de77d4b3 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/102212e4f0cc46e9830d3bb2de77d4b3 2024-11-28T07:43:44,550 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d2f35abaf31c4843b60e386826673034 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/A/d2f35abaf31c4843b60e386826673034 2024-11-28T07:43:44,551 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5195c97186874c12865019cf6aecb41d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/5195c97186874c12865019cf6aecb41d 2024-11-28T07:43:44,552 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dd4437c7af654fe9918f475cdadc456d to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/B/dd4437c7af654fe9918f475cdadc456d 2024-11-28T07:43:44,553 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/a7ca6110ef214fa38a529ec4e596af25 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/a7ca6110ef214fa38a529ec4e596af25 2024-11-28T07:43:44,554 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/b0f51d8f75c4443580e06d540d188ff2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/C/b0f51d8f75c4443580e06d540d188ff2 2024-11-28T07:43:44,556 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/recovered.edits/321.seqid to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2/recovered.edits/321.seqid 2024-11-28T07:43:44,556 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/default/TestAcidGuarantees/a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,556 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-28T07:43:44,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T07:43:44,557 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-28T07:43:44,559 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280f9b1d0b92d342a680a019f3a3fbcc77_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411280f9b1d0b92d342a680a019f3a3fbcc77_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,560 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282cfd645bc3904701a4692b900042aedc_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282cfd645bc3904701a4692b900042aedc_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282de3cfa17ee64216b1f7212357a2db75_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411282de3cfa17ee64216b1f7212357a2db75_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,561 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112830e7b40158e1424fb29c07c525532438_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112830e7b40158e1424fb29c07c525532438_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,562 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284d5e18c11cc3414496e102349742471b_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411284d5e18c11cc3414496e102349742471b_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,563 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128587a2cd5cdc8456ba77a8d70b15c0f63_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128587a2cd5cdc8456ba77a8d70b15c0f63_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112867450ed21c3144b4a5c919e0cd5faf5e_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112867450ed21c3144b4a5c919e0cd5faf5e_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,564 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128893c77fa46e34f0899ada142f04beb2c_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128893c77fa46e34f0899ada142f04beb2c_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,565 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288e9ed67d16bb49bfab7963a99c24d4f6_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411288e9ed67d16bb49bfab7963a99c24d4f6_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,566 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112893f5bed8774147fa8d7e662f13e482d3_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112893f5bed8774147fa8d7e662f13e482d3_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,567 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289c09cb722102496890e9d2b64a7b416f_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411289c09cb722102496890e9d2b64a7b416f_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,567 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a034561eced94f2bac2f1d1fa856ce1a_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a034561eced94f2bac2f1d1fa856ce1a_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,568 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a2f9ac6a08914538b5481f34d99aa9a9_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128a2f9ac6a08914538b5481f34d99aa9a9_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,569 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bbc03ef8aa8e4482a83585b317e78b1a_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128bbc03ef8aa8e4482a83585b317e78b1a_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,570 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4575e996bbf4ece9e87ac6403e041ba_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128d4575e996bbf4ece9e87ac6403e041ba_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,570 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128deff424f8337435bbb1803628782a79b_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128deff424f8337435bbb1803628782a79b_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,571 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f8b69d2eae99442ebdc058a36578f516_a4c1644781f0b1f4c5e05c56d56502e2 to hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241128f8b69d2eae99442ebdc058a36578f516_a4c1644781f0b1f4c5e05c56d56502e2 2024-11-28T07:43:44,572 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-28T07:43:44,574 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:44,575 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-28T07:43:44,577 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-28T07:43:44,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:44,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-28T07:43:44,578 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732779824578"}]},"ts":"9223372036854775807"} 2024-11-28T07:43:44,579 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-28T07:43:44,579 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a4c1644781f0b1f4c5e05c56d56502e2, NAME => 'TestAcidGuarantees,,1732779794411.a4c1644781f0b1f4c5e05c56d56502e2.', STARTKEY => '', ENDKEY => ''}] 2024-11-28T07:43:44,579 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-28T07:43:44,579 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732779824579"}]},"ts":"9223372036854775807"} 2024-11-28T07:43:44,581 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-28T07:43:44,581 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-28T07:43:44,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-28T07:43:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39063 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-28T07:43:44,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-28T07:43:44,651 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-28T07:43:44,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 247) - Thread LEAK? -, OpenFileDescriptor=459 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 303), ProcessCount=11 (was 11), AvailableMemoryMB=6067 (was 6195) 2024-11-28T07:43:44,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T07:43:44,661 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:43:44,661 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:43:44,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:44,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:44,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T07:43:44,661 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T07:43:44,661 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2051383393, stopped=false 2024-11-28T07:43:44,662 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f64cf8bdf19b,39063,1732779634677 2024-11-28T07:43:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:43:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T07:43:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:43:44,664 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:43:44,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:43:44,665 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T07:43:44,665 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:43:44,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:44,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:43:44,665 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f64cf8bdf19b,34437,1732779635619' ***** 2024-11-28T07:43:44,665 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T07:43:44,665 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T07:43:44,665 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T07:43:44,666 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T07:43:44,666 INFO [RS:0;f64cf8bdf19b:34437 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T07:43:44,666 INFO [RS:0;f64cf8bdf19b:34437 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T07:43:44,666 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(959): stopping server f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:44,666 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:43:44,666 INFO [RS:0;f64cf8bdf19b:34437 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f64cf8bdf19b:34437. 2024-11-28T07:43:44,666 DEBUG [RS:0;f64cf8bdf19b:34437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T07:43:44,666 DEBUG [RS:0;f64cf8bdf19b:34437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:44,667 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T07:43:44,667 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T07:43:44,667 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T07:43:44,667 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T07:43:44,667 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T07:43:44,667 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-28T07:43:44,668 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T07:43:44,668 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T07:43:44,668 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:44,668 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T07:43:44,668 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T07:43:44,668 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T07:43:44,668 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-28T07:43:44,691 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/info/fdf9bae3b750425091b25466b9e80e38 is 95, key is TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda./info:/1732779663358/DeleteFamily/seqid=0 2024-11-28T07:43:44,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742461_1637 (size=6514) 2024-11-28T07:43:44,737 INFO [regionserver/f64cf8bdf19b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:43:44,754 INFO [regionserver/f64cf8bdf19b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T07:43:44,754 INFO [regionserver/f64cf8bdf19b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T07:43:44,868 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:45,068 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:45,095 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/info/fdf9bae3b750425091b25466b9e80e38 2024-11-28T07:43:45,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-28T07:43:45,116 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/ns/4c0cb99b82624442a6fb432276c69f96 is 93, key is TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda./ns:/1732779663301/DeleteFamily/seqid=0 2024-11-28T07:43:45,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742462_1638 (size=5906) 2024-11-28T07:43:45,268 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:45,469 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:45,519 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/ns/4c0cb99b82624442a6fb432276c69f96 2024-11-28T07:43:45,536 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/rep_barrier/aa60dc7deef345ceb0ad5c82108b2de2 is 102, key is TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda./rep_barrier:/1732779663301/DeleteFamily/seqid=0 2024-11-28T07:43:45,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742463_1639 (size=6025) 2024-11-28T07:43:45,669 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:45,869 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:45,940 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/rep_barrier/aa60dc7deef345ceb0ad5c82108b2de2 2024-11-28T07:43:45,957 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/table/85e1469dc0a2424b977942147cdabaf7 is 96, key is TestAcidGuarantees,,1732779639239.e36af693bfc3c757988aba93aff1bdda./table:/1732779663301/DeleteFamily/seqid=0 2024-11-28T07:43:45,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742464_1640 (size=5936) 2024-11-28T07:43:46,072 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:46,272 DEBUG [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-28T07:43:46,361 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/table/85e1469dc0a2424b977942147cdabaf7 2024-11-28T07:43:46,364 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/info/fdf9bae3b750425091b25466b9e80e38 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/info/fdf9bae3b750425091b25466b9e80e38 2024-11-28T07:43:46,367 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/info/fdf9bae3b750425091b25466b9e80e38, entries=12, sequenceid=90, filesize=6.4 K 2024-11-28T07:43:46,368 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/ns/4c0cb99b82624442a6fb432276c69f96 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/ns/4c0cb99b82624442a6fb432276c69f96 2024-11-28T07:43:46,370 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/ns/4c0cb99b82624442a6fb432276c69f96, entries=8, sequenceid=90, filesize=5.8 K 2024-11-28T07:43:46,371 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/rep_barrier/aa60dc7deef345ceb0ad5c82108b2de2 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/rep_barrier/aa60dc7deef345ceb0ad5c82108b2de2 2024-11-28T07:43:46,373 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/rep_barrier/aa60dc7deef345ceb0ad5c82108b2de2, entries=6, sequenceid=90, filesize=5.9 K 2024-11-28T07:43:46,374 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/.tmp/table/85e1469dc0a2424b977942147cdabaf7 as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/table/85e1469dc0a2424b977942147cdabaf7 2024-11-28T07:43:46,377 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/table/85e1469dc0a2424b977942147cdabaf7, entries=7, sequenceid=90, filesize=5.8 K 2024-11-28T07:43:46,378 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false 2024-11-28T07:43:46,381 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-28T07:43:46,382 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T07:43:46,382 INFO [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T07:43:46,382 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732779824667Running coprocessor pre-close hooks at 1732779824668 (+1 ms)Disabling compacts and flushes for region at 1732779824668Disabling writes for close at 1732779824668Obtaining lock to block concurrent updates at 1732779824668Preparing flush snapshotting stores in 1588230740 at 1732779824668Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732779824669 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732779824669Flushing 1588230740/info: creating writer at 1732779824669Flushing 1588230740/info: appending metadata at 1732779824691 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732779824691Flushing 1588230740/ns: creating writer at 1732779825098 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732779825115 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732779825115Flushing 1588230740/rep_barrier: creating writer at 1732779825522 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732779825536 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732779825536Flushing 1588230740/table: creating writer at 1732779825943 (+407 ms)Flushing 1588230740/table: appending metadata at 1732779825957 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732779825957Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d6576de: reopening flushed file at 1732779826364 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2323e7b9: reopening flushed file at 1732779826367 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19bed58b: reopening flushed file at 1732779826370 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ea39004: reopening flushed file at 1732779826374 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false at 1732779826378 (+4 ms)Writing region close event to WAL at 1732779826379 (+1 ms)Running coprocessor post-close hooks at 1732779826382 (+3 ms)Closed at 1732779826382 2024-11-28T07:43:46,382 DEBUG [RS_CLOSE_META-regionserver/f64cf8bdf19b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T07:43:46,473 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(976): stopping server f64cf8bdf19b,34437,1732779635619; all regions closed. 2024-11-28T07:43:46,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741834_1010 (size=25344) 2024-11-28T07:43:46,479 DEBUG [RS:0;f64cf8bdf19b:34437 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/oldWALs 2024-11-28T07:43:46,479 INFO [RS:0;f64cf8bdf19b:34437 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C34437%2C1732779635619.meta:.meta(num 1732779638827) 2024-11-28T07:43:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741832_1008 (size=14716614) 2024-11-28T07:43:46,482 DEBUG [RS:0;f64cf8bdf19b:34437 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/oldWALs 2024-11-28T07:43:46,482 INFO [RS:0;f64cf8bdf19b:34437 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f64cf8bdf19b%2C34437%2C1732779635619:(num 1732779637874) 2024-11-28T07:43:46,482 DEBUG [RS:0;f64cf8bdf19b:34437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T07:43:46,482 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T07:43:46,482 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:43:46,482 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.ChoreService(370): Chore service for: regionserver/f64cf8bdf19b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T07:43:46,483 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:43:46,483 INFO [regionserver/f64cf8bdf19b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:43:46,483 INFO [RS:0;f64cf8bdf19b:34437 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34437 2024-11-28T07:43:46,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f64cf8bdf19b,34437,1732779635619 2024-11-28T07:43:46,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T07:43:46,487 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:43:46,488 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f64cf8bdf19b,34437,1732779635619] 2024-11-28T07:43:46,490 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f64cf8bdf19b,34437,1732779635619 already deleted, retry=false 2024-11-28T07:43:46,490 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f64cf8bdf19b,34437,1732779635619 expired; onlineServers=0 2024-11-28T07:43:46,490 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f64cf8bdf19b,39063,1732779634677' ***** 2024-11-28T07:43:46,490 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T07:43:46,491 INFO [M:0;f64cf8bdf19b:39063 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T07:43:46,491 INFO [M:0;f64cf8bdf19b:39063 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T07:43:46,491 DEBUG [M:0;f64cf8bdf19b:39063 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T07:43:46,491 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T07:43:46,491 DEBUG [M:0;f64cf8bdf19b:39063 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T07:43:46,491 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779637573 {}] cleaner.HFileCleaner(306): Exit Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.small.0-1732779637573,5,FailOnTimeoutGroup] 2024-11-28T07:43:46,491 DEBUG [master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779637572 {}] cleaner.HFileCleaner(306): Exit Thread[master/f64cf8bdf19b:0:becomeActiveMaster-HFileCleaner.large.0-1732779637572,5,FailOnTimeoutGroup] 2024-11-28T07:43:46,491 INFO [M:0;f64cf8bdf19b:39063 {}] hbase.ChoreService(370): Chore service for: master/f64cf8bdf19b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T07:43:46,491 INFO [M:0;f64cf8bdf19b:39063 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T07:43:46,491 DEBUG [M:0;f64cf8bdf19b:39063 {}] master.HMaster(1795): Stopping service threads 2024-11-28T07:43:46,491 INFO [M:0;f64cf8bdf19b:39063 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T07:43:46,491 INFO [M:0;f64cf8bdf19b:39063 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T07:43:46,492 ERROR [M:0;f64cf8bdf19b:39063 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:44369 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44369,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-28T07:43:46,492 INFO [M:0;f64cf8bdf19b:39063 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T07:43:46,492 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T07:43:46,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T07:43:46,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T07:43:46,493 DEBUG [M:0;f64cf8bdf19b:39063 {}] zookeeper.ZKUtil(347): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T07:43:46,493 WARN [M:0;f64cf8bdf19b:39063 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T07:43:46,493 INFO [M:0;f64cf8bdf19b:39063 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/.lastflushedseqids 2024-11-28T07:43:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742465_1641 (size=72) 2024-11-28T07:43:46,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:43:46,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1015d84d6970001, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:43:46,590 INFO [RS:0;f64cf8bdf19b:34437 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:43:46,590 INFO [RS:0;f64cf8bdf19b:34437 {}] regionserver.HRegionServer(1031): Exiting; stopping=f64cf8bdf19b,34437,1732779635619; zookeeper connection closed. 2024-11-28T07:43:46,590 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7fd992be {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7fd992be 2024-11-28T07:43:46,591 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T07:43:46,904 INFO [M:0;f64cf8bdf19b:39063 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T07:43:46,904 INFO [M:0;f64cf8bdf19b:39063 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T07:43:46,904 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T07:43:46,904 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:43:46,904 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:43:46,904 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T07:43:46,904 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:43:46,904 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=781.47 KB heapSize=957.83 KB 2024-11-28T07:43:46,920 DEBUG [M:0;f64cf8bdf19b:39063 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b4ce39f52244395a94288a0b069669a is 82, key is hbase:meta,,1/info:regioninfo/1732779638912/Put/seqid=0 2024-11-28T07:43:46,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742466_1642 (size=5672) 2024-11-28T07:43:47,323 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2175 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b4ce39f52244395a94288a0b069669a 2024-11-28T07:43:47,343 DEBUG [M:0;f64cf8bdf19b:39063 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e37cab3d5a12475497e9dcf5501be77b is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1732779797478/Put/seqid=0 2024-11-28T07:43:47,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742467_1643 (size=46395) 2024-11-28T07:43:47,346 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=780.91 KB at sequenceid=2175 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e37cab3d5a12475497e9dcf5501be77b 2024-11-28T07:43:47,349 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e37cab3d5a12475497e9dcf5501be77b 2024-11-28T07:43:47,364 DEBUG [M:0;f64cf8bdf19b:39063 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/13875ab03e0e41c4b5ecf810e14e575c is 69, key is f64cf8bdf19b,34437,1732779635619/rs:state/1732779637636/Put/seqid=0 2024-11-28T07:43:47,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073742468_1644 (size=5156) 2024-11-28T07:43:47,767 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2175 (bloomFilter=true), to=hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/13875ab03e0e41c4b5ecf810e14e575c 2024-11-28T07:43:47,771 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b4ce39f52244395a94288a0b069669a as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8b4ce39f52244395a94288a0b069669a 2024-11-28T07:43:47,774 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8b4ce39f52244395a94288a0b069669a, entries=8, sequenceid=2175, filesize=5.5 K 2024-11-28T07:43:47,774 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e37cab3d5a12475497e9dcf5501be77b as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e37cab3d5a12475497e9dcf5501be77b 2024-11-28T07:43:47,777 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e37cab3d5a12475497e9dcf5501be77b 2024-11-28T07:43:47,777 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e37cab3d5a12475497e9dcf5501be77b, entries=176, sequenceid=2175, filesize=45.3 K 2024-11-28T07:43:47,777 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/13875ab03e0e41c4b5ecf810e14e575c as hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/13875ab03e0e41c4b5ecf810e14e575c 2024-11-28T07:43:47,779 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44369/user/jenkins/test-data/e015ccd5-82f5-4e4b-c5bc-e11b95df7573/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/13875ab03e0e41c4b5ecf810e14e575c, entries=1, sequenceid=2175, filesize=5.0 K 2024-11-28T07:43:47,780 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(3140): Finished flush of dataSize ~781.47 KB/800222, heapSize ~957.53 KB/980512, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 876ms, sequenceid=2175, compaction requested=false 2024-11-28T07:43:47,781 INFO [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T07:43:47,782 DEBUG [M:0;f64cf8bdf19b:39063 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732779826904Disabling compacts and flushes for region at 1732779826904Disabling writes for close at 1732779826904Obtaining lock to block concurrent updates at 1732779826904Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732779826904Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=800222, getHeapSize=980752, getOffHeapSize=0, getCellsCount=2376 at 1732779826905 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732779826905Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732779826905Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732779826919 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732779826919Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732779827326 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732779827342 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732779827342Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732779827349 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732779827363 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732779827363Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6067b124: reopening flushed file at 1732779827770 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4a1e8d: reopening flushed file at 1732779827774 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cdaf7fb: reopening flushed file at 1732779827777 (+3 ms)Finished flush of dataSize ~781.47 KB/800222, heapSize ~957.53 KB/980512, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 876ms, sequenceid=2175, compaction requested=false at 1732779827780 (+3 ms)Writing region close event to WAL at 1732779827781 (+1 ms)Closed at 1732779827781 2024-11-28T07:43:47,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36239 is added to blk_1073741830_1006 (size=942820) 2024-11-28T07:43:47,784 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T07:43:47,784 INFO [M:0;f64cf8bdf19b:39063 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T07:43:47,785 INFO [M:0;f64cf8bdf19b:39063 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39063 2024-11-28T07:43:47,785 INFO [M:0;f64cf8bdf19b:39063 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T07:43:47,887 INFO [M:0;f64cf8bdf19b:39063 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T07:43:47,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:43:47,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39063-0x1015d84d6970000, quorum=127.0.0.1:53972, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T07:43:47,892 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T07:43:47,894 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T07:43:47,894 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T07:43:47,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T07:43:47,894 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/hadoop.log.dir/,STOPPED} 2024-11-28T07:43:47,896 WARN [BP-123607211-172.17.0.2-1732779631377 heartbeating to localhost/127.0.0.1:44369 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T07:43:47,896 WARN [BP-123607211-172.17.0.2-1732779631377 heartbeating to localhost/127.0.0.1:44369 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-123607211-172.17.0.2-1732779631377 (Datanode Uuid 03c664ec-2777-4875-979d-320ad1fc2a6f) service to localhost/127.0.0.1:44369 2024-11-28T07:43:47,896 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T07:43:47,896 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T07:43:47,898 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/data/data1/current/BP-123607211-172.17.0.2-1732779631377 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T07:43:47,899 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/cluster_e38abcf3-7756-c34e-ff06-c3174d2c49b4/data/data2/current/BP-123607211-172.17.0.2-1732779631377 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T07:43:47,899 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T07:43:47,905 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T07:43:47,906 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T07:43:47,906 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T07:43:47,906 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T07:43:47,906 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/96e884be-49e5-a90a-0bf7-69accd4b419c/hadoop.log.dir/,STOPPED} 2024-11-28T07:43:47,922 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T07:43:48,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-28T07:43:48,135 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T07:43:48,137 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null